imports: try to use global imports unless it is a layering violation

To minimize the impact, do imports that violate the layering at runtime instead
of at import time.

kallithea/config/middleware/pygrack.py

@@ -1,229 +1,227 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.middleware.pygrack
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Python implementation of git-http-backend's Smart HTTP protocol
 
 Based on original code from git_http_backend.py project.
 
 Copyright (c) 2010 Daniel Dotsenko <[e-mail]>
 Copyright (c) 2012 Marcin Kuzminski <[e-mail]>
 
 This file was forked by the Kallithea project in July 2014.
 """
 
 import logging
 import os
 import socket
 import traceback
 
 from dulwich.server import update_server_info
 from dulwich.web import GunzipFilter, LimitedInputFilter
 from webob import Request, Response, exc
 
 import kallithea
 from kallithea.lib.utils2 import ascii_bytes
-from kallithea.lib.vcs import subprocessio
+from kallithea.lib.vcs import get_repo, subprocessio
 
 
 log = logging.getLogger(__name__)
 
 
 class FileWrapper(object):
 
     def __init__(self, fd, content_length):
         self.fd = fd
         self.content_length = content_length
         self.remain = content_length
 
     def read(self, size):
         if size <= self.remain:
             try:
                 data = self.fd.read(size)
             except socket.error:
                 raise IOError(self)
             self.remain -= size
         elif self.remain:
             data = self.fd.read(self.remain)
             self.remain = 0
         else:
             data = None
         return data
 
     def __repr__(self):
         return '<FileWrapper %s len: %s, read: %s>' % (
             self.fd, self.content_length, self.content_length - self.remain
         )
 
 
 class GitRepository(object):
     git_folder_signature = set(['config', 'head', 'info', 'objects', 'refs'])
     commands = ['git-upload-pack', 'git-receive-pack']
 
     def __init__(self, repo_name, content_path):
         files = set([f.lower() for f in os.listdir(content_path)])
         if not (self.git_folder_signature.intersection(files)
                 == self.git_folder_signature):
             raise OSError('%s missing git signature' % content_path)
         self.content_path = content_path
         self.valid_accepts = ['application/x-%s-result' %
                               c for c in self.commands]
         self.repo_name = repo_name
 
     def _get_fixedpath(self, path):
         """
         Small fix for repo_path
 
         :param path:
         """
         assert path.startswith('/' + self.repo_name + '/')
         return path[len(self.repo_name) + 2:].strip('/')
 
     def inforefs(self, req, environ):
         """
         WSGI Response producer for HTTP GET Git Smart
         HTTP /info/refs request.
         """
 
         git_command = req.GET.get('service')
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPMethodNotAllowed()
 
         # From Documentation/technical/http-protocol.txt shipped with Git:
         #
         # Clients MUST verify the first pkt-line is `# service=$servicename`.
         # Servers MUST set $servicename to be the request parameter value.
         # Servers SHOULD include an LF at the end of this line.
         # Clients MUST ignore an LF at the end of the line.
         #
         #  smart_reply     =  PKT-LINE("# service=$servicename" LF)
         #                     ref_list
         #                     "0000"
         server_advert = '# service=%s\n' % git_command
         packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
         _git_path = kallithea.CONFIG.get('git_path', 'git')
         cmd = [_git_path, git_command[4:],
                '--stateless-rpc', '--advertise-refs', self.content_path]
         log.debug('handling cmd %s', cmd)
         try:
             out = subprocessio.SubprocessIOChunker(cmd,
                 starting_values=[ascii_bytes(packet_len + server_advert + '0000')]
             )
         except EnvironmentError as e:
             log.error(traceback.format_exc())
             raise exc.HTTPExpectationFailed()
         resp = Response()
         resp.content_type = 'application/x-%s-advertisement' % git_command
         resp.charset = None
         resp.app_iter = out
         return resp
 
     def backend(self, req, environ):
         """
         WSGI Response producer for HTTP POST Git Smart HTTP requests.
         Reads commands and data from HTTP POST's body.
         returns an iterator obj with contents of git command's
         response to stdout
         """
         _git_path = kallithea.CONFIG.get('git_path', 'git')
         git_command = self._get_fixedpath(req.path_info)
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPMethodNotAllowed()
 
         if 'CONTENT_LENGTH' in environ:
             inputstream = FileWrapper(environ['wsgi.input'],
                                       req.content_length)
         else:
             inputstream = environ['wsgi.input']
 
         gitenv = dict(os.environ)
         # forget all configs
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         cmd = [_git_path, git_command[4:], '--stateless-rpc', self.content_path]
         log.debug('handling cmd %s', cmd)
         try:
             out = subprocessio.SubprocessIOChunker(
                 cmd,
                 inputstream=inputstream,
                 env=gitenv,
                 cwd=self.content_path,
             )
         except EnvironmentError as e:
             log.error(traceback.format_exc())
             raise exc.HTTPExpectationFailed()
 
         if git_command in ['git-receive-pack']:
             # updating refs manually after each push.
             # Needed for pre-1.7.0.4 git clients using regular HTTP mode.
-
-            from kallithea.lib.vcs import get_repo
             repo = get_repo(self.content_path)
             if repo:
                 update_server_info(repo._repo)
 
         resp = Response()
         resp.content_type = 'application/x-%s-result' % git_command
         resp.charset = None
         resp.app_iter = out
         return resp
 
     def __call__(self, environ, start_response):
         req = Request(environ)
         _path = self._get_fixedpath(req.path_info)
         if _path.startswith('info/refs'):
             app = self.inforefs
         elif req.accept.acceptable_offers(self.valid_accepts):
             app = self.backend
         try:
             resp = app(req, environ)
         except exc.HTTPException as e:
             resp = e
             log.error(traceback.format_exc())
         except Exception as e:
             log.error(traceback.format_exc())
             resp = exc.HTTPInternalServerError()
         return resp(environ, start_response)
 
 
 class GitDirectory(object):
 
     def __init__(self, repo_root, repo_name):
         repo_location = os.path.join(repo_root, repo_name)
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
 
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
 
     def __call__(self, environ, start_response):
         content_path = self.content_path
         try:
             app = GitRepository(self.repo_name, content_path)
         except (AssertionError, OSError):
             content_path = os.path.join(content_path, '.git')
             if os.path.isdir(content_path):
                 app = GitRepository(self.repo_name, content_path)
             else:
                 return exc.HTTPNotFound()(environ, start_response)
         return app(environ, start_response)
 
 
 def make_wsgi_app(repo_name, repo_root):
     app = GitDirectory(repo_root, repo_name)
     return GunzipFilter(LimitedInputFilter(app))

kallithea/controllers/admin/settings.py

@@ -1,412 +1,411 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.admin.settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 settings controller for Kallithea admin
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jul 14, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import traceback
 
 import formencode
 from formencode import htmlfill
 from tg import config, request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPFound
 
+import kallithea
 from kallithea.lib import webutils
 from kallithea.lib.auth import HasPermissionAnyDecorator, LoginRequired
 from kallithea.lib.base import BaseController, render
 from kallithea.lib.celerylib import tasks
 from kallithea.lib.utils import repo2db_mapper, set_app_settings
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.vcs import VCSError
 from kallithea.lib.webutils import url
 from kallithea.model import db, meta
 from kallithea.model.forms import ApplicationSettingsForm, ApplicationUiSettingsForm, ApplicationVisualisationForm
 from kallithea.model.notification import EmailNotificationModel
 from kallithea.model.scm import ScmModel
 
 
 log = logging.getLogger(__name__)
 
 
 class SettingsController(BaseController):
 
     @LoginRequired(allow_default_user=True)
     def _before(self, *args, **kwargs):
         super(SettingsController, self)._before(*args, **kwargs)
 
     def _get_hg_ui_settings(self):
         ret = db.Ui.query().all()
 
         settings = {}
         for each in ret:
             k = each.ui_section + '_' + each.ui_key
             v = each.ui_value
             if k == 'paths_/':
                 k = 'paths_root_path'
 
             k = k.replace('.', '_')
 
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
 
             settings[k] = v
         return settings
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_vcs(self):
         c.active = 'vcs'
         if request.POST:
             application_form = ApplicationUiSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                      render('admin/settings/settings.html'),
                      defaults=errors.value,
                      errors=errors.error_dict or {},
                      prefix_error=False,
                      encoding="UTF-8",
                      force_defaults=False)
 
             try:
                 if c.visual.allow_repo_location_change:
                     sett = db.Ui.get_by_key('paths', '/')
                     sett.ui_value = form_result['paths_root_path']
 
                 # HOOKS
                 sett = db.Ui.get_by_key('hooks', db.Ui.HOOK_UPDATE)
                 sett.ui_active = form_result['hooks_changegroup_update']
 
                 sett = db.Ui.get_by_key('hooks', db.Ui.HOOK_REPO_SIZE)
                 sett.ui_active = form_result['hooks_changegroup_repo_size']
 
                 ## EXTENSIONS
                 sett = db.Ui.get_or_create('extensions', 'largefiles')
                 sett.ui_active = form_result['extensions_largefiles']
 
 #                sett = db.Ui.get_or_create('extensions', 'hggit')
 #                sett.ui_active = form_result['extensions_hggit']
 
                 meta.Session().commit()
 
                 webutils.flash(_('Updated VCS settings'), category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 webutils.flash(_('Error occurred while updating '
                           'application settings'), category='error')
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_mapping(self):
         c.active = 'mapping'
         if request.POST:
             rm_obsolete = request.POST.get('destroy', False)
             install_git_hooks = request.POST.get('hooks', False)
             overwrite_git_hooks = request.POST.get('hooks_overwrite', False)
             invalidate_cache = request.POST.get('invalidate', False)
             log.debug('rescanning repo location with destroy obsolete=%s, '
                       'install git hooks=%s and '
                       'overwrite git hooks=%s' % (rm_obsolete, install_git_hooks, overwrite_git_hooks))
 
             filesystem_repos = ScmModel().repo_scan()
             added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,
                                             install_git_hooks=install_git_hooks,
                                             user=request.authuser.username,
                                             overwrite_git_hooks=overwrite_git_hooks)
             added_msg = webutils.HTML(', ').join(
                 webutils.link_to(safe_str(repo_name), webutils.url('summary_home', repo_name=repo_name)) for repo_name in added
             ) or '-'
             removed_msg = webutils.HTML(', ').join(
                 safe_str(repo_name) for repo_name in removed
             ) or '-'
             webutils.flash(webutils.HTML(_('Repositories successfully rescanned. Added: %s. Removed: %s.')) %
                     (added_msg, removed_msg), category='success')
 
             if invalidate_cache:
                 log.debug('invalidating all repositories cache')
                 i = 0
                 for repo in db.Repository.query():
                     try:
                         ScmModel().mark_for_invalidation(repo.repo_name)
                         i += 1
                     except VCSError as e:
                         log.warning('VCS error invalidating %s: %s', repo.repo_name, e)
                 webutils.flash(_('Invalidated %s repositories') % i, category='success')
 
             raise HTTPFound(location=url('admin_settings_mapping'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_global(self):
         c.active = 'global'
         if request.POST:
             application_form = ApplicationSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('admin/settings/settings.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
 
             try:
                 for setting in (
                     'title',
                     'realm',
                     'ga_code',
                     'captcha_public_key',
                     'captcha_private_key',
                 ):
                     db.Setting.create_or_update(setting, form_result[setting])
 
                 meta.Session().commit()
                 set_app_settings(config)
                 webutils.flash(_('Updated application settings'), category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 webutils.flash(_('Error occurred while updating '
                           'application settings'),
                           category='error')
 
             raise HTTPFound(location=url('admin_settings_global'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_visual(self):
         c.active = 'visual'
         if request.POST:
             application_form = ApplicationVisualisationForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('admin/settings/settings.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
 
             try:
                 settings = [
                     ('show_public_icon', 'show_public_icon', 'bool'),
                     ('show_private_icon', 'show_private_icon', 'bool'),
                     ('stylify_metalabels', 'stylify_metalabels', 'bool'),
                     ('repository_fields', 'repository_fields', 'bool'),
                     ('dashboard_items', 'dashboard_items', 'int'),
                     ('admin_grid_items', 'admin_grid_items', 'int'),
                     ('show_version', 'show_version', 'bool'),
                     ('use_gravatar', 'use_gravatar', 'bool'),
                     ('gravatar_url', 'gravatar_url', 'unicode'),
                     ('clone_uri_tmpl', 'clone_uri_tmpl', 'unicode'),
                     ('clone_ssh_tmpl', 'clone_ssh_tmpl', 'unicode'),
                 ]
                 for setting, form_key, type_ in settings:
                     db.Setting.create_or_update(setting, form_result[form_key], type_)
 
                 meta.Session().commit()
                 set_app_settings(config)
                 webutils.flash(_('Updated visualisation settings'),
                         category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 webutils.flash(_('Error occurred during updating '
                           'visualisation settings'),
                         category='error')
 
             raise HTTPFound(location=url('admin_settings_visual'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_email(self):
         c.active = 'email'
         if request.POST:
             test_email = request.POST.get('test_email')
             test_email_subj = 'Kallithea test email'
             test_body = ('Kallithea Email test, '
                                'Kallithea version: %s' % c.kallithea_version)
             if not test_email:
                 webutils.flash(_('Please enter email address'), category='error')
                 raise HTTPFound(location=url('admin_settings_email'))
 
             test_email_txt_body = EmailNotificationModel() \
                 .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
                                 'txt', body=test_body)
             test_email_html_body = EmailNotificationModel() \
                 .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
                                 'html', body=test_body)
 
             recipients = [test_email] if test_email else None
 
             tasks.send_email(recipients, test_email_subj,
                              test_email_txt_body, test_email_html_body)
 
             webutils.flash(_('Send email task created'), category='success')
             raise HTTPFound(location=url('admin_settings_email'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
-        import kallithea
         c.ini = kallithea.CONFIG
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_hooks(self):
         c.active = 'hooks'
         if request.POST:
             if c.visual.allow_custom_hooks_settings:
                 ui_key = request.POST.get('new_hook_ui_key')
                 ui_value = request.POST.get('new_hook_ui_value')
 
                 hook_id = request.POST.get('hook_id')
 
                 try:
                     ui_key = ui_key and ui_key.strip()
                     if ui_key in (x.ui_key for x in db.Ui.get_custom_hooks()):
                         webutils.flash(_('Hook already exists'), category='error')
                     elif ui_key in (x.ui_key for x in db.Ui.get_builtin_hooks()):
                         webutils.flash(_('Builtin hooks are read-only. Please use another hook name.'), category='error')
                     elif ui_value and ui_key:
                         db.Ui.create_or_update_hook(ui_key, ui_value)
                         webutils.flash(_('Added new hook'), category='success')
                     elif hook_id:
                         db.Ui.delete(hook_id)
                         meta.Session().commit()
 
                     # check for edits
                     update = False
                     _d = request.POST.dict_of_lists()
                     for k, v, ov in zip(_d.get('hook_ui_key', []),
                                         _d.get('hook_ui_value_new', []),
                                         _d.get('hook_ui_value', [])):
                         if v != ov:
                             db.Ui.create_or_update_hook(k, v)
                             update = True
 
                     if update:
                         webutils.flash(_('Updated hooks'), category='success')
                     meta.Session().commit()
                 except Exception:
                     log.error(traceback.format_exc())
                     webutils.flash(_('Error occurred during hook creation'),
                             category='error')
 
                 raise HTTPFound(location=url('admin_settings_hooks'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         c.hooks = db.Ui.get_builtin_hooks()
         c.custom_hooks = db.Ui.get_custom_hooks()
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_search(self):
         c.active = 'search'
         if request.POST:
             repo_location = self._get_hg_ui_settings()['paths_root_path']
             full_index = request.POST.get('full_index', False)
             tasks.whoosh_index(repo_location, full_index)
             webutils.flash(_('Whoosh reindex task scheduled'), category='success')
             raise HTTPFound(location=url('admin_settings_search'))
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAnyDecorator('hg.admin')
     def settings_system(self):
         c.active = 'system'
 
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
 
-        import kallithea
         c.ini = kallithea.CONFIG
         server_info = db.Setting.get_server_info()
         for key, val in server_info.items():
             setattr(c, key, val)
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)

kallithea/controllers/login.py

@@ -1,257 +1,256 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.login
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Login controller for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 22, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import logging
 import re
 
 import formencode
 from formencode import htmlfill
 from tg import request, session
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPFound
 
 from kallithea.lib import webutils
 from kallithea.lib.auth import AuthUser, HasPermissionAnyDecorator
 from kallithea.lib.base import BaseController, log_in_user, render
 from kallithea.lib.exceptions import UserCreationError
+from kallithea.lib.recaptcha import submit
 from kallithea.lib.webutils import url
 from kallithea.model import db, meta
 from kallithea.model.forms import LoginForm, PasswordResetConfirmationForm, PasswordResetRequestForm, RegisterForm
 from kallithea.model.user import UserModel
 
 
 log = logging.getLogger(__name__)
 
 
 class LoginController(BaseController):
 
     def _validate_came_from(self, came_from,
             _re=re.compile(r"/(?!/)[-!#$%&'()*+,./:;=?@_~0-9A-Za-z]*$")):
         """Return True if came_from is valid and can and should be used.
 
         Determines if a URI reference is valid and relative to the origin;
         or in RFC 3986 terms, whether it matches this production:
 
           origin-relative-ref = path-absolute [ "?" query ] [ "#" fragment ]
 
         with the exception that '%' escapes are not validated and '#' is
         allowed inside the fragment part.
         """
         return _re.match(came_from) is not None
 
     def index(self):
         c.came_from = request.GET.get('came_from', '')
         if c.came_from:
             if not self._validate_came_from(c.came_from):
                 log.error('Invalid came_from (not server-relative): %r', c.came_from)
                 raise HTTPBadRequest()
         else:
             c.came_from = url('home')
 
         if request.POST:
             # import Login Form validator class
             login_form = LoginForm()()
             try:
                 # login_form will check username/password using ValidAuth and report failure to the user
                 c.form_result = login_form.to_python(dict(request.POST))
                 username = c.form_result['username']
                 user = db.User.get_by_username_or_email(username)
                 assert user is not None  # the same user get just passed in the form validation
             except formencode.Invalid as errors:
                 defaults = errors.value
                 # remove password from filling in form again
                 defaults.pop('password', None)
                 return htmlfill.render(
                     render('/login.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             except UserCreationError as e:
                 # container auth or other auth functions that create users on
                 # the fly can throw this exception signaling that there's issue
                 # with user creation, explanation should be provided in
                 # Exception itself
                 webutils.flash(e, 'error')
             else:
                 # login_form already validated the password - now set the session cookie accordingly
                 auth_user = log_in_user(user, c.form_result['remember'], is_external_auth=False, ip_addr=request.ip_addr)
                 if auth_user:
                     raise HTTPFound(location=c.came_from)
                 webutils.flash(_('Authentication failed.'), 'error')
         else:
             # redirect if already logged in
             if not request.authuser.is_anonymous:
                 raise HTTPFound(location=c.came_from)
             # continue to show login to default user
 
         return render('/login.html')
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate',
                                'hg.register.manual_activate')
     def register(self):
         def_user_perms = AuthUser(dbuser=db.User.get_default_user()).global_permissions
         c.auto_active = 'hg.register.auto_activate' in def_user_perms
 
         settings = db.Setting.get_app_settings()
         captcha_private_key = settings.get('captcha_private_key')
         c.captcha_active = bool(captcha_private_key)
         c.captcha_public_key = settings.get('captcha_public_key')
 
         if request.POST:
             register_form = RegisterForm()()
             try:
                 form_result = register_form.to_python(dict(request.POST))
                 form_result['active'] = c.auto_active
 
                 if c.captcha_active:
-                    from kallithea.lib.recaptcha import submit
                     response = submit(request.POST.get('g-recaptcha-response'),
                                       private_key=captcha_private_key,
                                       remoteip=request.ip_addr)
                     if not response.is_valid:
                         _value = form_result
                         _msg = _('Bad captcha')
                         error_dict = {'recaptcha_field': _msg}
                         raise formencode.Invalid(_msg, _value, None,
                                                  error_dict=error_dict)
 
                 UserModel().create_registration(form_result)
                 webutils.flash(_('You have successfully registered with %s') % (c.site_name or 'Kallithea'),
                         category='success')
                 meta.Session().commit()
                 raise HTTPFound(location=url('login_home'))
 
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('/register.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             except UserCreationError as e:
                 # container auth or other auth functions that create users on
                 # the fly can throw this exception signaling that there's issue
                 # with user creation, explanation should be provided in
                 # Exception itself
                 webutils.flash(e, 'error')
 
         return render('/register.html')
 
     def password_reset(self):
         settings = db.Setting.get_app_settings()
         captcha_private_key = settings.get('captcha_private_key')
         c.captcha_active = bool(captcha_private_key)
         c.captcha_public_key = settings.get('captcha_public_key')
 
         if request.POST:
             password_reset_form = PasswordResetRequestForm()()
             try:
                 form_result = password_reset_form.to_python(dict(request.POST))
                 if c.captcha_active:
-                    from kallithea.lib.recaptcha import submit
                     response = submit(request.POST.get('g-recaptcha-response'),
                                       private_key=captcha_private_key,
                                       remoteip=request.ip_addr)
                     if not response.is_valid:
                         _value = form_result
                         _msg = _('Bad captcha')
                         error_dict = {'recaptcha_field': _msg}
                         raise formencode.Invalid(_msg, _value, None,
                                                  error_dict=error_dict)
                 redirect_link = UserModel().send_reset_password_email(form_result)
                 webutils.flash(_('A password reset confirmation code has been sent'),
                             category='success')
                 raise HTTPFound(location=redirect_link)
 
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('/password_reset.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
 
         return render('/password_reset.html')
 
     def password_reset_confirmation(self):
         # This controller handles both GET and POST requests, though we
         # only ever perform the actual password change on POST (since
         # GET requests are not allowed to have side effects, and do not
         # receive automatic CSRF protection).
 
         # The template needs the email address outside of the form.
         c.email = request.params.get('email')
         c.timestamp = request.params.get('timestamp') or ''
         c.token = request.params.get('token') or ''
         if not request.POST:
             return render('/password_reset_confirmation.html')
 
         form = PasswordResetConfirmationForm()()
         try:
             form_result = form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('/password_reset_confirmation.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding='UTF-8')
 
         if not UserModel().verify_reset_password_token(
             form_result['email'],
             form_result['timestamp'],
             form_result['token'],
         ):
             return htmlfill.render(
                 render('/password_reset_confirmation.html'),
                 defaults=form_result,
                 errors={'token': _('Invalid password reset token')},
                 prefix_error=False,
                 encoding='UTF-8')
 
         UserModel().reset_password(form_result['email'], form_result['password'])
         webutils.flash(_('Successfully updated password'), category='success')
         raise HTTPFound(location=url('login_home'))
 
     def logout(self):
         session.delete()
         log.info('Logging out and deleting session for user')
         raise HTTPFound(location=url('home'))
 
     def session_csrf_secret_token(self):
         """Return the CSRF protection token for the session - just like it
         could have been screen scraped from a page with a form.
         Only intended for testing but might also be useful for other kinds
         of automation.
         """
         return webutils.session_csrf_secret_token()

kallithea/controllers/routing.py

@@ -1,774 +1,773 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Routes configuration
 
 The more specific and detailed routes should be defined first so they
 may take precedent over the more generic routes. For more information
 refer to the routes manual at http://routes.groovie.org/docs/
 """
 
 import routes
 
 import kallithea
+from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
 from kallithea.lib.utils2 import safe_str
 
 
 class Mapper(routes.Mapper):
     """
     Subclassed Mapper with routematch patched to decode "unicode" str url to
     *real* unicode str before applying matches and invoking controller methods.
     """
 
     def routematch(self, url=None, environ=None):
         """
         routematch that also decode url from "fake bytes" to real unicode
         string before matching and invoking controllers.
         """
         # Process url like get_path_info does ... but PATH_INFO has already
         # been retrieved from environ and is passed, so - let's just use that
         # instead.
         url = safe_str(url.encode('latin1'))
         return super().routematch(url=url, environ=environ)
 
 
 def make_map(config):
     """Create, configure and return the routes Mapper"""
     rmap = Mapper(directory=config['paths']['controllers'],
                   always_scan=config['debug'])
     rmap.minimization = False
     rmap.explicit = False
 
-    from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
-
     def check_repo(environ, match_dict):
         """
         Check for valid repository for proper 404 handling.
         Also, a bit of side effect modifying match_dict ...
         """
         if match_dict.get('f_path'):
             # fix for multiple initial slashes that causes errors
             match_dict['f_path'] = match_dict['f_path'].lstrip('/')
 
         return is_valid_repo(match_dict['repo_name'], config['base_path'])
 
     def check_group(environ, match_dict):
         """
         check for valid repository group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repo_group_name = match_dict.get('group_name')
         return is_valid_repo_group(repo_group_name, config['base_path'])
 
     def check_group_skip_path(environ, match_dict):
         """
         check for valid repository group for proper 404 handling, but skips
         verification of existing path
 
         :param environ:
         :param match_dict:
         """
         repo_group_name = match_dict.get('group_name')
         return is_valid_repo_group(repo_group_name, config['base_path'],
                                    skip_path_check=True)
 
     def check_user_group(environ, match_dict):
         """
         check for valid user group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         return True
 
     def check_int(environ, match_dict):
         return match_dict.get('id').isdigit()
 
     #==========================================================================
     # CUSTOM ROUTES HERE
     #==========================================================================
 
     # MAIN PAGE
     rmap.connect('home', '/', controller='home')
     rmap.connect('about', '/about', controller='home', action='about')
     rmap.redirect('/favicon.ico', '/images/favicon.ico')
     rmap.connect('repo_switcher_data', '/_repos', controller='home',
                  action='repo_switcher_data')
     rmap.connect('users_and_groups_data', '/_users_and_groups', controller='home',
                  action='users_and_groups_data')
 
     rmap.connect('rst_help',
                  "http://docutils.sourceforge.net/docs/user/rst/quickref.html",
                  _static=True)
     rmap.connect('kallithea_project_url', "https://kallithea-scm.org/", _static=True)
     rmap.connect('issues_url', 'https://bitbucket.org/conservancy/kallithea/issues', _static=True)
 
     # ADMIN REPOSITORY ROUTES
     ADMIN_PREFIX = kallithea.ADMIN_PREFIX
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repos') as m:
         m.connect("repos", "/repos",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("repos", "/repos",
                   conditions=dict(method=["GET"]))
         m.connect("new_repo", "/create_repository",
                   action="create_repository", conditions=dict(method=["GET"]))
         m.connect("update_repo", "/repos/{repo_name:.*?}",
                   action="update", conditions=dict(method=["POST"],
                   function=check_repo))
         m.connect("delete_repo", "/repos/{repo_name:.*?}/delete",
                   action="delete", conditions=dict(method=["POST"]))
 
     # ADMIN REPOSITORY GROUPS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repo_groups') as m:
         m.connect("repos_groups", "/repo_groups",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("repos_groups", "/repo_groups",
                   conditions=dict(method=["GET"]))
         m.connect("new_repos_group", "/repo_groups/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_repos_group", "/repo_groups/{group_name:.*?}",
                   action="update", conditions=dict(method=["POST"],
                                                    function=check_group))
 
         m.connect("repos_group", "/repo_groups/{group_name:.*?}",
                   action="show", conditions=dict(method=["GET"],
                                                  function=check_group))
 
         # EXTRAS REPO GROUP ROUTES
         m.connect("edit_repo_group", "/repo_groups/{group_name:.*?}/edit",
                   action="edit",
                   conditions=dict(method=["GET"], function=check_group))
 
         m.connect("edit_repo_group_advanced", "/repo_groups/{group_name:.*?}/edit/advanced",
                   action="edit_repo_group_advanced",
                   conditions=dict(method=["GET"], function=check_group))
 
         m.connect("edit_repo_group_perms", "/repo_groups/{group_name:.*?}/edit/permissions",
                   action="edit_repo_group_perms",
                   conditions=dict(method=["GET"], function=check_group))
         m.connect("edit_repo_group_perms_update", "/repo_groups/{group_name:.*?}/edit/permissions",
                   action="update_perms",
                   conditions=dict(method=["POST"], function=check_group))
         m.connect("edit_repo_group_perms_delete", "/repo_groups/{group_name:.*?}/edit/permissions/delete",
                   action="delete_perms",
                   conditions=dict(method=["POST"], function=check_group))
 
         m.connect("delete_repo_group", "/repo_groups/{group_name:.*?}/delete",
                   action="delete", conditions=dict(method=["POST"],
                                                    function=check_group_skip_path))
 
     # ADMIN USER ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/users') as m:
         m.connect("new_user", "/users/new",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("users", "/users",
                   conditions=dict(method=["GET"]))
         m.connect("formatted_users", "/users.{format}",
                   conditions=dict(method=["GET"]))
         m.connect("new_user", "/users/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_user", "/users/{id}",
                   action="update", conditions=dict(method=["POST"]))
         m.connect("delete_user", "/users/{id}/delete",
                   action="delete", conditions=dict(method=["POST"]))
         m.connect("edit_user", "/users/{id}/edit",
                   action="edit", conditions=dict(method=["GET"]))
 
         # EXTRAS USER ROUTES
         m.connect("edit_user_advanced", "/users/{id}/edit/advanced",
                   action="edit_advanced", conditions=dict(method=["GET"]))
 
         m.connect("edit_user_api_keys", "/users/{id}/edit/api_keys",
                   action="edit_api_keys", conditions=dict(method=["GET"]))
         m.connect("edit_user_api_keys_update", "/users/{id}/edit/api_keys",
                   action="add_api_key", conditions=dict(method=["POST"]))
         m.connect("edit_user_api_keys_delete", "/users/{id}/edit/api_keys/delete",
                   action="delete_api_key", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_ssh_keys", "/users/{id}/edit/ssh_keys",
                   action="edit_ssh_keys", conditions=dict(method=["GET"]))
         m.connect("edit_user_ssh_keys", "/users/{id}/edit/ssh_keys",
                   action="ssh_keys_add", conditions=dict(method=["POST"]))
         m.connect("edit_user_ssh_keys_delete", "/users/{id}/edit/ssh_keys/delete",
                   action="ssh_keys_delete", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_perms", "/users/{id}/edit/permissions",
                   action="edit_perms", conditions=dict(method=["GET"]))
         m.connect("edit_user_perms_update", "/users/{id}/edit/permissions",
                   action="update_perms", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_emails", "/users/{id}/edit/emails",
                   action="edit_emails", conditions=dict(method=["GET"]))
         m.connect("edit_user_emails_update", "/users/{id}/edit/emails",
                   action="add_email", conditions=dict(method=["POST"]))
         m.connect("edit_user_emails_delete", "/users/{id}/edit/emails/delete",
                   action="delete_email", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_ips", "/users/{id}/edit/ips",
                   action="edit_ips", conditions=dict(method=["GET"]))
         m.connect("edit_user_ips_update", "/users/{id}/edit/ips",
                   action="add_ip", conditions=dict(method=["POST"]))
         m.connect("edit_user_ips_delete", "/users/{id}/edit/ips/delete",
                   action="delete_ip", conditions=dict(method=["POST"]))
 
     # ADMIN USER GROUPS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/user_groups') as m:
         m.connect("users_groups", "/user_groups",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("users_groups", "/user_groups",
                   conditions=dict(method=["GET"]))
         m.connect("new_users_group", "/user_groups/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_users_group", "/user_groups/{id}",
                   action="update", conditions=dict(method=["POST"]))
         m.connect("delete_users_group", "/user_groups/{id}/delete",
                   action="delete", conditions=dict(method=["POST"]))
         m.connect("edit_users_group", "/user_groups/{id}/edit",
                   action="edit", conditions=dict(method=["GET"]),
                   function=check_user_group)
 
         # EXTRAS USER GROUP ROUTES
         m.connect("edit_user_group_default_perms", "/user_groups/{id}/edit/default_perms",
                   action="edit_default_perms", conditions=dict(method=["GET"]))
         m.connect("edit_user_group_default_perms_update", "/user_groups/{id}/edit/default_perms",
                   action="update_default_perms", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_group_perms", "/user_groups/{id}/edit/perms",
                   action="edit_perms", conditions=dict(method=["GET"]))
         m.connect("edit_user_group_perms_update", "/user_groups/{id}/edit/perms",
                   action="update_perms", conditions=dict(method=["POST"]))
         m.connect("edit_user_group_perms_delete", "/user_groups/{id}/edit/perms/delete",
                   action="delete_perms", conditions=dict(method=["POST"]))
 
         m.connect("edit_user_group_advanced", "/user_groups/{id}/edit/advanced",
                   action="edit_advanced", conditions=dict(method=["GET"]))
 
         m.connect("edit_user_group_members", "/user_groups/{id}/edit/members",
                   action="edit_members", conditions=dict(method=["GET"]))
 
     # ADMIN PERMISSIONS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/permissions') as m:
         m.connect("admin_permissions", "/permissions",
                   action="permission_globals", conditions=dict(method=["POST"]))
         m.connect("admin_permissions", "/permissions",
                   action="permission_globals", conditions=dict(method=["GET"]))
 
         m.connect("admin_permissions_ips", "/permissions/ips",
                   action="permission_ips", conditions=dict(method=["GET"]))
 
         m.connect("admin_permissions_perms", "/permissions/perms",
                   action="permission_perms", conditions=dict(method=["GET"]))
 
     # ADMIN DEFAULTS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/defaults') as m:
         m.connect('defaults', '/defaults')
         m.connect('defaults_update', 'defaults/{id}/update',
                   action="update", conditions=dict(method=["POST"]))
 
     # ADMIN AUTH SETTINGS
     rmap.connect('auth_settings', '%s/auth' % ADMIN_PREFIX,
                  controller='admin/auth_settings', action='auth_settings',
                  conditions=dict(method=["POST"]))
     rmap.connect('auth_home', '%s/auth' % ADMIN_PREFIX,
                  controller='admin/auth_settings')
 
     # ADMIN SETTINGS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/settings') as m:
         m.connect("admin_settings", "/settings",
                   action="settings_vcs", conditions=dict(method=["POST"]))
         m.connect("admin_settings", "/settings",
                   action="settings_vcs", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_mapping", "/settings/mapping",
                   action="settings_mapping", conditions=dict(method=["POST"]))
         m.connect("admin_settings_mapping", "/settings/mapping",
                   action="settings_mapping", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_global", "/settings/global",
                   action="settings_global", conditions=dict(method=["POST"]))
         m.connect("admin_settings_global", "/settings/global",
                   action="settings_global", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_visual", "/settings/visual",
                   action="settings_visual", conditions=dict(method=["POST"]))
         m.connect("admin_settings_visual", "/settings/visual",
                   action="settings_visual", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_email", "/settings/email",
                   action="settings_email", conditions=dict(method=["POST"]))
         m.connect("admin_settings_email", "/settings/email",
                   action="settings_email", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_hooks", "/settings/hooks",
                   action="settings_hooks", conditions=dict(method=["POST"]))
         m.connect("admin_settings_hooks_delete", "/settings/hooks/delete",
                   action="settings_hooks", conditions=dict(method=["POST"]))
         m.connect("admin_settings_hooks", "/settings/hooks",
                   action="settings_hooks", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_search", "/settings/search",
                   action="settings_search", conditions=dict(method=["POST"]))
         m.connect("admin_settings_search", "/settings/search",
                   action="settings_search", conditions=dict(method=["GET"]))
 
         m.connect("admin_settings_system", "/settings/system",
                   action="settings_system", conditions=dict(method=["POST"]))
         m.connect("admin_settings_system", "/settings/system",
                   action="settings_system", conditions=dict(method=["GET"]))
 
     # ADMIN MY ACCOUNT
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/my_account') as m:
 
         m.connect("my_account", "/my_account",
                   action="my_account", conditions=dict(method=["GET"]))
         m.connect("my_account", "/my_account",
                   action="my_account", conditions=dict(method=["POST"]))
 
         m.connect("my_account_password", "/my_account/password",
                   action="my_account_password", conditions=dict(method=["GET"]))
         m.connect("my_account_password", "/my_account/password",
                   action="my_account_password", conditions=dict(method=["POST"]))
 
         m.connect("my_account_repos", "/my_account/repos",
                   action="my_account_repos", conditions=dict(method=["GET"]))
 
         m.connect("my_account_watched", "/my_account/watched",
                   action="my_account_watched", conditions=dict(method=["GET"]))
 
         m.connect("my_account_perms", "/my_account/perms",
                   action="my_account_perms", conditions=dict(method=["GET"]))
 
         m.connect("my_account_emails", "/my_account/emails",
                   action="my_account_emails", conditions=dict(method=["GET"]))
         m.connect("my_account_emails", "/my_account/emails",
                   action="my_account_emails_add", conditions=dict(method=["POST"]))
         m.connect("my_account_emails_delete", "/my_account/emails/delete",
                   action="my_account_emails_delete", conditions=dict(method=["POST"]))
 
         m.connect("my_account_api_keys", "/my_account/api_keys",
                   action="my_account_api_keys", conditions=dict(method=["GET"]))
         m.connect("my_account_api_keys", "/my_account/api_keys",
                   action="my_account_api_keys_add", conditions=dict(method=["POST"]))
         m.connect("my_account_api_keys_delete", "/my_account/api_keys/delete",
                   action="my_account_api_keys_delete", conditions=dict(method=["POST"]))
 
         m.connect("my_account_ssh_keys", "/my_account/ssh_keys",
                   action="my_account_ssh_keys", conditions=dict(method=["GET"]))
         m.connect("my_account_ssh_keys", "/my_account/ssh_keys",
                   action="my_account_ssh_keys_add", conditions=dict(method=["POST"]))
         m.connect("my_account_ssh_keys_delete", "/my_account/ssh_keys/delete",
                   action="my_account_ssh_keys_delete", conditions=dict(method=["POST"]))
 
     # ADMIN GIST
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/gists') as m:
         m.connect("gists", "/gists",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("gists", "/gists",
                   conditions=dict(method=["GET"]))
         m.connect("new_gist", "/gists/new",
                   action="new", conditions=dict(method=["GET"]))
 
         m.connect("gist_delete", "/gists/{gist_id}/delete",
                   action="delete", conditions=dict(method=["POST"]))
         m.connect("edit_gist", "/gists/{gist_id}/edit",
                   action="edit", conditions=dict(method=["GET", "POST"]))
         m.connect("edit_gist_check_revision", "/gists/{gist_id}/edit/check_revision",
                   action="check_revision", conditions=dict(method=["POST"]))
 
         m.connect("gist", "/gists/{gist_id}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("gist_rev", "/gists/{gist_id}/{revision}",
                   revision="tip",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_gist", "/gists/{gist_id}/{revision}/{format}",
                   revision="tip",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_gist_file", "/gists/{gist_id}/{revision}/{format}/{f_path:.*}",
                   revision='tip',
                   action="show", conditions=dict(method=["GET"]))
 
     # ADMIN MAIN PAGES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/admin') as m:
         m.connect('admin_home', '')
         m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9. _-]*}',
                   action='add_repo')
     #==========================================================================
     # API V2
     #==========================================================================
     with rmap.submapper(path_prefix=ADMIN_PREFIX, controller='api/api',
                         action='_dispatch') as m:
         m.connect('api', '/api')
 
     # USER JOURNAL
     rmap.connect('journal', '%s/journal' % ADMIN_PREFIX,
                  controller='journal')
     rmap.connect('journal_rss', '%s/journal/rss' % ADMIN_PREFIX,
                  controller='journal', action='journal_rss')
     rmap.connect('journal_atom', '%s/journal/atom' % ADMIN_PREFIX,
                  controller='journal', action='journal_atom')
 
     rmap.connect('public_journal', '%s/public_journal' % ADMIN_PREFIX,
                  controller='journal', action="public_journal")
 
     rmap.connect('public_journal_rss', '%s/public_journal/rss' % ADMIN_PREFIX,
                  controller='journal', action="public_journal_rss")
 
     rmap.connect('public_journal_rss_old', '%s/public_journal_rss' % ADMIN_PREFIX,
                  controller='journal', action="public_journal_rss")
 
     rmap.connect('public_journal_atom',
                  '%s/public_journal/atom' % ADMIN_PREFIX, controller='journal',
                  action="public_journal_atom")
 
     rmap.connect('public_journal_atom_old',
                  '%s/public_journal_atom' % ADMIN_PREFIX, controller='journal',
                  action="public_journal_atom")
 
     rmap.connect('toggle_following', '%s/toggle_following' % ADMIN_PREFIX,
                  controller='journal', action='toggle_following',
                  conditions=dict(method=["POST"]))
 
     # SEARCH
     rmap.connect('search', '%s/search' % ADMIN_PREFIX, controller='search',)
     rmap.connect('search_repo_admin', '%s/search/{repo_name:.*}' % ADMIN_PREFIX,
                  controller='search',
                  conditions=dict(function=check_repo))
     rmap.connect('search_repo', '/{repo_name:.*?}/search',
                  controller='search',
                  conditions=dict(function=check_repo),
                  )
 
     # LOGIN/LOGOUT/REGISTER/SIGN IN
     rmap.connect('session_csrf_secret_token', '%s/session_csrf_secret_token' % ADMIN_PREFIX, controller='login', action='session_csrf_secret_token')
     rmap.connect('login_home', '%s/login' % ADMIN_PREFIX, controller='login')
     rmap.connect('logout_home', '%s/logout' % ADMIN_PREFIX, controller='login',
                  action='logout')
 
     rmap.connect('register', '%s/register' % ADMIN_PREFIX, controller='login',
                  action='register')
 
     rmap.connect('reset_password', '%s/password_reset' % ADMIN_PREFIX,
                  controller='login', action='password_reset')
 
     rmap.connect('reset_password_confirmation',
                  '%s/password_reset_confirmation' % ADMIN_PREFIX,
                  controller='login', action='password_reset_confirmation')
 
     # FEEDS
     rmap.connect('rss_feed_home', '/{repo_name:.*?}/feed/rss',
                 controller='feed', action='rss',
                 conditions=dict(function=check_repo))
 
     rmap.connect('atom_feed_home', '/{repo_name:.*?}/feed/atom',
                 controller='feed', action='atom',
                 conditions=dict(function=check_repo))
 
     #==========================================================================
     # REPOSITORY ROUTES
     #==========================================================================
     rmap.connect('repo_creating_home', '/{repo_name:.*?}/repo_creating',
                 controller='admin/repos', action='repo_creating')
     rmap.connect('repo_check_home', '/{repo_name:.*?}/repo_check_creating',
                 controller='admin/repos', action='repo_check')
 
     rmap.connect('summary_home', '/{repo_name:.*?}',
                 controller='summary',
                 conditions=dict(function=check_repo))
 
     # must be here for proper group/repo catching
     rmap.connect('repos_group_home', '/{group_name:.*}',
                 controller='admin/repo_groups', action="show_by_name",
                 conditions=dict(function=check_group))
     rmap.connect('repo_stats_home', '/{repo_name:.*?}/statistics',
                 controller='summary', action='statistics',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_size', '/{repo_name:.*?}/repo_size',
                 controller='summary', action='repo_size',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_refs_data', '/{repo_name:.*?}/refs-data',
                  controller='home', action='repo_refs_data')
 
     rmap.connect('changeset_home', '/{repo_name:.*?}/changeset/{revision:.*}',
                 controller='changeset', revision='tip',
                 conditions=dict(function=check_repo))
     rmap.connect('changeset_children', '/{repo_name:.*?}/changeset_children/{revision}',
                 controller='changeset', revision='tip', action="changeset_children",
                 conditions=dict(function=check_repo))
     rmap.connect('changeset_parents', '/{repo_name:.*?}/changeset_parents/{revision}',
                 controller='changeset', revision='tip', action="changeset_parents",
                 conditions=dict(function=check_repo))
 
     # repo edit options
     rmap.connect("edit_repo", "/{repo_name:.*?}/settings",
                  controller='admin/repos', action="edit",
                  conditions=dict(method=["GET"], function=check_repo))
 
     rmap.connect("edit_repo_perms", "/{repo_name:.*?}/settings/permissions",
                  controller='admin/repos', action="edit_permissions",
                  conditions=dict(method=["GET"], function=check_repo))
     rmap.connect("edit_repo_perms_update", "/{repo_name:.*?}/settings/permissions",
                  controller='admin/repos', action="edit_permissions_update",
                  conditions=dict(method=["POST"], function=check_repo))
     rmap.connect("edit_repo_perms_revoke", "/{repo_name:.*?}/settings/permissions/delete",
                  controller='admin/repos', action="edit_permissions_revoke",
                  conditions=dict(method=["POST"], function=check_repo))
 
     rmap.connect("edit_repo_fields", "/{repo_name:.*?}/settings/fields",
                  controller='admin/repos', action="edit_fields",
                  conditions=dict(method=["GET"], function=check_repo))
     rmap.connect('create_repo_fields', "/{repo_name:.*?}/settings/fields/new",
                  controller='admin/repos', action="create_repo_field",
                  conditions=dict(method=["POST"], function=check_repo))
     rmap.connect('delete_repo_fields', "/{repo_name:.*?}/settings/fields/{field_id}/delete",
                  controller='admin/repos', action="delete_repo_field",
                  conditions=dict(method=["POST"], function=check_repo))
 
     rmap.connect("edit_repo_advanced", "/{repo_name:.*?}/settings/advanced",
                  controller='admin/repos', action="edit_advanced",
                  conditions=dict(method=["GET"], function=check_repo))
 
     rmap.connect("edit_repo_advanced_journal", "/{repo_name:.*?}/settings/advanced/journal",
                  controller='admin/repos', action="edit_advanced_journal",
                  conditions=dict(method=["POST"], function=check_repo))
 
     rmap.connect("edit_repo_advanced_fork", "/{repo_name:.*?}/settings/advanced/fork",
                  controller='admin/repos', action="edit_advanced_fork",
                  conditions=dict(method=["POST"], function=check_repo))
 
     rmap.connect("edit_repo_remote", "/{repo_name:.*?}/settings/remote",
                  controller='admin/repos', action="edit_remote",
                  conditions=dict(method=["GET"], function=check_repo))
     rmap.connect("edit_repo_remote_update", "/{repo_name:.*?}/settings/remote",
                  controller='admin/repos', action="edit_remote",
                  conditions=dict(method=["POST"], function=check_repo))
 
     rmap.connect("edit_repo_statistics", "/{repo_name:.*?}/settings/statistics",
                  controller='admin/repos', action="edit_statistics",
                  conditions=dict(method=["GET"], function=check_repo))
     rmap.connect("edit_repo_statistics_update", "/{repo_name:.*?}/settings/statistics",
                  controller='admin/repos', action="edit_statistics",
                  conditions=dict(method=["POST"], function=check_repo))
 
     # still working url for backward compat.
     rmap.connect('raw_changeset_home_depraced',
                  '/{repo_name:.*?}/raw-changeset/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions=dict(function=check_repo))
 
     ## new URLs
     rmap.connect('changeset_raw_home',
                  '/{repo_name:.*?}/changeset-diff/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_patch_home',
                  '/{repo_name:.*?}/changeset-patch/{revision}',
                  controller='changeset', action='changeset_patch',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_download_home',
                  '/{repo_name:.*?}/changeset-download/{revision}',
                  controller='changeset', action='changeset_download',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_comment',
                  '/{repo_name:.*?}/changeset-comment/{revision}',
                 controller='changeset', revision='tip', action='comment',
                 conditions=dict(function=check_repo))
 
     rmap.connect('changeset_comment_delete',
                  '/{repo_name:.*?}/changeset-comment/{comment_id}/delete',
                 controller='changeset', action='delete_comment',
                 conditions=dict(function=check_repo, method=["POST"]))
 
     rmap.connect('changeset_info', '/changeset_info/{repo_name:.*?}/{revision}',
                  controller='changeset', action='changeset_info')
 
     rmap.connect('compare_home',
                  '/{repo_name:.*?}/compare',
                  controller='compare',
                  conditions=dict(function=check_repo))
 
     rmap.connect('compare_url',
                  '/{repo_name:.*?}/compare/{org_ref_type}@{org_ref_name:.*?}...{other_ref_type}@{other_ref_name:.*?}',
                  controller='compare', action='compare',
                  conditions=dict(function=check_repo),
                  requirements=dict(
                             org_ref_type='(branch|book|tag|rev|__other_ref_type__)',
                             other_ref_type='(branch|book|tag|rev|__org_ref_type__)')
                  )
 
     rmap.connect('pullrequest_home',
                  '/{repo_name:.*?}/pull-request/new', controller='pullrequests',
                  conditions=dict(function=check_repo,
                                                  method=["GET"]))
 
     rmap.connect('pullrequest_repo_info',
                  '/{repo_name:.*?}/pull-request-repo-info',
                  controller='pullrequests', action='repo_info',
                  conditions=dict(function=check_repo, method=["GET"]))
 
     rmap.connect('pullrequest',
                  '/{repo_name:.*?}/pull-request/new', controller='pullrequests',
                  action='create', conditions=dict(function=check_repo,
                                                   method=["POST"]))
 
     rmap.connect('pullrequest_show',
                  '/{repo_name:.*?}/pull-request/{pull_request_id:\\d+}{extra:(/.*)?}', extra='',
                  controller='pullrequests',
                  action='show', conditions=dict(function=check_repo,
                                                 method=["GET"]))
     rmap.connect('pullrequest_post',
                  '/{repo_name:.*?}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='post', conditions=dict(function=check_repo,
                                                 method=["POST"]))
     rmap.connect('pullrequest_delete',
                  '/{repo_name:.*?}/pull-request/{pull_request_id}/delete',
                  controller='pullrequests',
                  action='delete', conditions=dict(function=check_repo,
                                                   method=["POST"]))
 
     rmap.connect('pullrequest_show_all',
                  '/{repo_name:.*?}/pull-request',
                  controller='pullrequests',
                  action='show_all', conditions=dict(function=check_repo,
                                                 method=["GET"]))
 
     rmap.connect('my_pullrequests',
                  '/my_pullrequests',
                  controller='pullrequests',
                  action='show_my', conditions=dict(method=["GET"]))
 
     rmap.connect('pullrequest_comment',
                  '/{repo_name:.*?}/pull-request-comment/{pull_request_id}',
                  controller='pullrequests',
                  action='comment', conditions=dict(function=check_repo,
                                                 method=["POST"]))
 
     rmap.connect('pullrequest_comment_delete',
                  '/{repo_name:.*?}/pull-request-comment/{comment_id}/delete',
                 controller='pullrequests', action='delete_comment',
                 conditions=dict(function=check_repo, method=["POST"]))
 
     rmap.connect('summary_home_summary', '/{repo_name:.*?}/summary',
                 controller='summary', conditions=dict(function=check_repo))
 
     rmap.connect('changelog_home', '/{repo_name:.*?}/changelog',
                 controller='changelog', conditions=dict(function=check_repo))
 
     rmap.connect('changelog_file_home', '/{repo_name:.*?}/changelog/{revision}/{f_path:.*}',
                 controller='changelog',
                 conditions=dict(function=check_repo))
 
     rmap.connect('changelog_details', '/{repo_name:.*?}/changelog_details/{cs}',
                 controller='changelog', action='changelog_details',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_home', '/{repo_name:.*?}/files/{revision}/{f_path:.*}',
                 controller='files', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_home_nopath', '/{repo_name:.*?}/files/{revision}',
                 controller='files', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_history_home',
                  '/{repo_name:.*?}/history/{revision}/{f_path:.*}',
                  controller='files', action='history', revision='tip', f_path='',
                  conditions=dict(function=check_repo))
 
     rmap.connect('files_authors_home',
                  '/{repo_name:.*?}/authors/{revision}/{f_path:.*}',
                  controller='files', action='authors', revision='tip', f_path='',
                  conditions=dict(function=check_repo))
 
     rmap.connect('files_diff_home', '/{repo_name:.*?}/diff/{f_path:.*}',
                 controller='files', action='diff', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_diff_2way_home', '/{repo_name:.*?}/diff-2way/{f_path:.+}',
                 controller='files', action='diff_2way', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_rawfile_home',
                  '/{repo_name:.*?}/rawfile/{revision}/{f_path:.*}',
                  controller='files', action='rawfile', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_raw_home',
                  '/{repo_name:.*?}/raw/{revision}/{f_path:.*}',
                  controller='files', action='raw', revision='tip', f_path='',
                  conditions=dict(function=check_repo))
 
     rmap.connect('files_annotate_home',
                  '/{repo_name:.*?}/annotate/{revision}/{f_path:.*}',
                  controller='files', revision='tip',
                  f_path='', annotate='1', conditions=dict(function=check_repo))
 
     rmap.connect('files_edit_home',
                  '/{repo_name:.*?}/edit/{revision}/{f_path:.*}',
                  controller='files', action='edit', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_add_home',
                  '/{repo_name:.*?}/add/{revision}/{f_path:.*}',
                  controller='files', action='add', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_delete_home',
                  '/{repo_name:.*?}/delete/{revision}/{f_path:.*}',
                  controller='files', action='delete', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_archive_home', '/{repo_name:.*?}/archive/{fname}',
                 controller='files', action='archivefile',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_nodelist_home',
                  '/{repo_name:.*?}/nodelist/{revision}/{f_path:.*}',
                 controller='files', action='nodelist',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_fork_create_home', '/{repo_name:.*?}/fork',
                 controller='forks', action='fork_create',
                 conditions=dict(function=check_repo, method=["POST"]))
 
     rmap.connect('repo_fork_home', '/{repo_name:.*?}/fork',
                 controller='forks', action='fork',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_forks_home', '/{repo_name:.*?}/forks',
                  controller='forks', action='forks',
                  conditions=dict(function=check_repo))
 
     rmap.connect('repo_followers_home', '/{repo_name:.*?}/followers',
                  controller='followers', action='followers',
                  conditions=dict(function=check_repo))
 
     return rmap

kallithea/lib/helpers.py

@@ -1,1166 +1,1163 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import hashlib
 import json
 import logging
 import re
 import textwrap
 import urllib.parse
 
 from beaker.cache import cache_region
 from pygments import highlight as code_highlight
 from pygments.formatters.html import HtmlFormatter
+from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 
 import kallithea
 from kallithea.lib.annotate import annotate_highlight
 #==============================================================================
 # PERMS
 #==============================================================================
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel
 from kallithea.lib.diffs import BIN_FILENODE, CHMOD_FILENODE, DEL_FILENODE, MOD_FILENODE, NEW_FILENODE, RENAMED_FILENODE
 from kallithea.lib.markup_renderer import url_re
 from kallithea.lib.pygmentsutils import get_custom_lexer
 from kallithea.lib.utils2 import (MENTIONS_REGEX, AttributeDict, age, asbool, credentials_filter, fmt_date, link_to_ref, safe_bytes, safe_int, safe_str,
                                   shorter, time_to_datetime)
 from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from kallithea.lib.vcs.utils import author_email, author_name
 from kallithea.lib.webutils import (HTML, Option, canonical_url, checkbox, chop_at, end_form, escape, form, format_byte_size, hidden, html_escape, link_to,
                                     literal, password, pop_flash_messages, radio, reset, safeid, select, session_csrf_secret_name, session_csrf_secret_token,
                                     submit, text, textarea, truncate, url, wrap_paragraphs)
 from kallithea.model import db
 from kallithea.model.changeset_status import ChangesetStatusModel
 
 
 # mute pyflakes "imported but unused"
 # from webutils
 assert Option
 assert canonical_url
 assert checkbox
 assert chop_at
 assert end_form
 assert form
 assert format_byte_size
 assert hidden
 assert password
 assert pop_flash_messages
 assert radio
 assert reset
 assert safeid
 assert select
 assert session_csrf_secret_name
 assert session_csrf_secret_token
 assert submit
 assert text
 assert textarea
 assert wrap_paragraphs
 # from kallithea.lib.auth
 assert HasPermissionAny
 assert HasRepoGroupPermissionLevel
 assert HasRepoPermissionLevel
 # from utils2
 assert age
 assert fmt_date
 assert link_to_ref
 assert shorter
 assert time_to_datetime
 # from vcs
 assert EmptyChangeset
 
 
 log = logging.getLogger(__name__)
 
 
 def js(value):
     """Convert Python value to the corresponding JavaScript representation.
 
     This is necessary to safely insert arbitrary values into HTML <script>
     sections e.g. using Mako template expression substitution.
 
     Note: Rather than using this function, it's preferable to avoid the
     insertion of values into HTML <script> sections altogether. Instead,
     data should (to the extent possible) be passed to JavaScript using
     data attributes or AJAX calls, eliminating the need for JS specific
     escaping.
 
     Note: This is not safe for use in attributes (e.g. onclick), because
     quotes are not escaped.
 
     Because the rules for parsing <script> varies between XHTML (where
     normal rules apply for any special characters) and HTML (where
     entities are not interpreted, but the literal string "</script>"
     is forbidden), the function ensures that the result never contains
     '&', '<' and '>', thus making it safe in both those contexts (but
     not in attributes).
     """
     return literal(
         ('(' + json.dumps(value) + ')')
         # In JSON, the following can only appear in string literals.
         .replace('&', r'\x26')
         .replace('<', r'\x3c')
         .replace('>', r'\x3e')
     )
 
 
 def jshtml(val):
     """HTML escapes a string value, then converts the resulting string
     to its corresponding JavaScript representation (see `js`).
 
     This is used when a plain-text string (possibly containing special
     HTML characters) will be used by a script in an HTML context (e.g.
     element.innerHTML or jQuery's 'html' method).
 
     If in doubt, err on the side of using `jshtml` over `js`, since it's
     better to escape too much than too little.
     """
     return js(escape(val))
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and revision
     it's safe to use in urls
     """
     return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])
 
 
 def get_ignore_whitespace_diff(GET):
     """Return true if URL requested whitespace to be ignored"""
     return bool(GET.get('ignorews'))
 
 
 def ignore_whitespace_link(GET, anchor=None):
     """Return snippet with link to current URL with whitespace ignoring toggled"""
     params = dict(GET)  # ignoring duplicates
     if get_ignore_whitespace_diff(GET):
         params.pop('ignorews')
         title = _("Show whitespace changes")
     else:
         params['ignorews'] = '1'
         title = _("Ignore whitespace changes")
     params['anchor'] = anchor
     return link_to(
         literal('<i class="icon-strike"></i>'),
         url.current(**params),
         title=title,
         **{'data-toggle': 'tooltip'})
 
 
 def get_diff_context_size(GET):
     """Return effective context size requested in URL"""
     return safe_int(GET.get('context'), default=3)
 
 
 def increase_context_link(GET, anchor=None):
     """Return snippet with link to current URL with double context size"""
     context = get_diff_context_size(GET) * 2
     params = dict(GET)  # ignoring duplicates
     params['context'] = str(context)
     params['anchor'] = anchor
     return link_to(
         literal('<i class="icon-sort"></i>'),
         url.current(**params),
         title=_('Increase diff context to %(num)s lines') % {'num': context},
         **{'data-toggle': 'tooltip'})
 
 
 class _FilesBreadCrumbs(object):
 
     def __call__(self, repo_name, rev, paths):
         url_l = [link_to(repo_name, url('files_home',
                                         repo_name=repo_name,
                                         revision=rev, f_path=''),
                          class_='ypjax-link')]
         paths_l = paths.split('/')
         for cnt, p in enumerate(paths_l):
             if p != '':
                 url_l.append(link_to(p,
                                      url('files_home',
                                          repo_name=repo_name,
                                          revision=rev,
                                          f_path='/'.join(paths_l[:cnt + 1])
                                          ),
                                      class_='ypjax-link'
                                      )
                              )
 
         return literal('/'.join(url_l))
 
 
 files_breadcrumbs = _FilesBreadCrumbs()
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<span id="L%s">%s</span>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         inner_lines = []
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             inner_lines.append(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv">'
                       '<pre>' + ls + '</pre></div></td>'
                       '<td id="hlcode" class="code">')
         yield 0, ''.join(inner_lines)
         yield 0, '</td></tr></table>'
 
 
 _whitespace_re = re.compile(r'(\t)|( )(?=\n|</div>)')
 
 
 def _markup_whitespace(m):
     groups = m.groups()
     if groups[0]:
         return '<u>\t</u>'
     if groups[1]:
         return ' <i></i>'
 
 
 def markup_whitespace(s):
     return _whitespace_re.sub(_markup_whitespace, s)
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return literal(markup_whitespace(
         code_highlight(safe_str(filenode.content), lexer, CodeHtmlFormatter(**kwargs))))
 
 
 def hsv_to_rgb(h, s, v):
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def gen_color(n=10000):
     """generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _unused in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, 0.95, 0.95]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield [str(int(x * 256)) for x in RGB_tuple]
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
     cgenerator = gen_color()
     color_dict = {}
 
     def get_color_string(cs):
         if cs in color_dict:
             col = color_dict[cs]
         else:
             col = color_dict[cs] = next(cgenerator)
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(changeset):
         author = escape(changeset.author)
         date = changeset.date
         message = escape(changeset.message)
         tooltip_html = ("<b>Author:</b> %s<br/>"
                         "<b>Date:</b> %s</b><br/>"
                         "<b>Message:</b> %s") % (author, date, message)
 
         lnk_format = show_id(changeset)
         uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=repo_name,
                     revision=changeset.raw_id),
                 style=get_color_string(changeset.raw_id),
                 **{'data-toggle': 'popover',
                    'data-content': tooltip_html}
               )
 
         uri += '\n'
         return uri
 
     return literal(markup_whitespace(annotate_highlight(filenode, url_func, **kwargs)))
 
 
 def capitalize(x):
     return x.capitalize()
 
 def short_id(x):
     return x[:12]
 
 def hide_credentials(x):
     return ''.join(credentials_filter(x))
 
 
 def show_id(cs):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param cs: changeset instance
     """
     def_len = safe_int(kallithea.CONFIG.get('show_sha_length', 12))
     show_rev = asbool(kallithea.CONFIG.get('show_revision_number', False))
 
     raw_id = cs.raw_id[:def_len]
     if show_rev:
         return 'r%s:%s' % (cs.revision, raw_id)
     else:
         return raw_id
 
 
 def is_git(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'git'
 
 
 def is_hg(repository):
     if hasattr(repository, 'alias'):
         _type = repository.alias
     elif hasattr(repository, 'repo_type'):
         _type = repository.repo_type
     else:
         _type = repository
     return _type == 'hg'
 
 
 @cache_region('long_term', 'user_attr_or_none')
 def user_attr_or_none(author, show_attr):
     """Try to match email part of VCS committer string with a local user and return show_attr
     - or return None if user not found"""
     email = author_email(author)
     if email:
         user = db.User.get_by_email(email)
         if user is not None:
             return getattr(user, show_attr)
     return None
 
 
 def email_or_none(author):
     """Try to match email part of VCS committer string with a local user.
     Return primary email of user, email part of the specified author name, or None."""
     if not author:
         return None
     email = user_attr_or_none(author, 'email')
     if email is not None:
         return email # always use user's main email address - not necessarily the one used to find user
 
     # extract email from the commit string
     email = author_email(author)
     if email:
         return email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def person(author, show_attr="username"):
     """Find the user identified by 'author', return one of the users attributes,
     default to the username attribute, None if there is no user"""
     # if author is already an instance use it for extraction
     if isinstance(author, db.User):
         return getattr(author, show_attr)
 
     value = user_attr_or_none(author, show_attr)
     if value is not None:
         return value
 
     # Still nothing?  Just pass back the author name if any, else the email
     return author_name(author) or author_email(author)
 
 
 def person_by_id(id_, show_attr="username"):
     # maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = db.User.get(id_)
         if user is not None:
             return getattr(user, show_attr)
     return id_
 
 
 def boolicon(value):
     """Returns boolean value of a value, represented as small html image of true/false
     icons
 
     :param value: value
     """
 
     if value:
         return HTML.tag('i', class_="icon-ok")
     else:
         return HTML.tag('i', class_="icon-minus-circled")
 
 
 def action_parser(user_log, feed=False, parse_cs=False):
     """
     This helper will action_map the specified string action into translated
     fancy names with icons and links
 
     :param user_log: user log instance
     :param feed: use output for feeds (no html and fancy icons)
     :param parse_cs: parse Changesets into VCS instances
     """
 
     action = user_log.action
     action_params = ' '
 
     x = action.split(':')
 
     if len(x) > 1:
         action, action_params = x
 
     def get_cs_links():
         revs_limit = 3  # display this amount always
         revs_top_limit = 50  # show upto this amount of changesets hidden
         revs_ids = action_params.split(',')
         deleted = user_log.repository is None
         if deleted:
             return ','.join(revs_ids)
 
         repo_name = user_log.repository.repo_name
 
         def lnk(rev, repo_name):
             lazy_cs = False
             title_ = None
             url_ = '#'
             if isinstance(rev, BaseChangeset) or isinstance(rev, AttributeDict):
                 if rev.op and rev.ref_name:
                     if rev.op == 'delete_branch':
                         lbl = _('Deleted branch: %s') % rev.ref_name
                     elif rev.op == 'tag':
                         lbl = _('Created tag: %s') % rev.ref_name
                     else:
                         lbl = 'Unknown operation %s' % rev.op
                 else:
                     lazy_cs = True
                     lbl = rev.short_id[:8]
                     url_ = url('changeset_home', repo_name=repo_name,
                                revision=rev.raw_id)
             else:
                 # changeset cannot be found - it might have been stripped or removed
                 lbl = rev[:12]
                 title_ = _('Changeset %s not found') % lbl
             if parse_cs:
                 return link_to(lbl, url_, title=title_, **{'data-toggle': 'tooltip'})
             return link_to(lbl, url_, class_='lazy-cs' if lazy_cs else '',
                            **{'data-raw_id': rev.raw_id, 'data-repo_name': repo_name})
 
         def _get_op(rev_txt):
             _op = None
             _name = rev_txt
             if len(rev_txt.split('=>')) == 2:
                 _op, _name = rev_txt.split('=>')
             return _op, _name
 
         revs = []
         if len([v for v in revs_ids if v != '']) > 0:
             repo = None
             for rev in revs_ids[:revs_top_limit]:
                 _op, _name = _get_op(rev)
 
                 # we want parsed changesets, or new log store format is bad
                 if parse_cs:
                     try:
                         if repo is None:
                             repo = user_log.repository.scm_instance
                         _rev = repo.get_changeset(rev)
                         revs.append(_rev)
                     except ChangesetDoesNotExistError:
                         log.error('cannot find revision %s in this repo', rev)
                         revs.append(rev)
                 else:
                     _rev = AttributeDict({
                         'short_id': rev[:12],
                         'raw_id': rev,
                         'message': '',
                         'op': _op,
                         'ref_name': _name
                     })
                     revs.append(_rev)
         cs_links = [" " + ', '.join(
             [lnk(rev, repo_name) for rev in revs[:revs_limit]]
         )]
         _op1, _name1 = _get_op(revs_ids[0])
         _op2, _name2 = _get_op(revs_ids[-1])
 
         _rev = '%s...%s' % (_name1, _name2)
 
         compare_view = (
             ' <div class="compare_view" data-toggle="tooltip" title="%s">'
             '<a href="%s">%s</a> </div>' % (
                 _('Show all combined changesets %s->%s') % (
                     revs_ids[0][:12], revs_ids[-1][:12]
                 ),
                 url('changeset_home', repo_name=repo_name,
                     revision=_rev
                 ),
                 _('Compare view')
             )
         )
 
         # if we have exactly one more than normally displayed
         # just display it, takes less space than displaying
         # "and 1 more revisions"
         if len(revs_ids) == revs_limit + 1:
             cs_links.append(", " + lnk(revs[revs_limit], repo_name))
 
         # hidden-by-default ones
         if len(revs_ids) > revs_limit + 1:
             uniq_id = revs_ids[0]
             html_tmpl = (
                 '<span> %s <a class="show_more" id="_%s" '
                 'href="#more">%s</a> %s</span>'
             )
             if not feed:
                 cs_links.append(html_tmpl % (
                       _('and'),
                       uniq_id, _('%s more') % (len(revs_ids) - revs_limit),
                       _('revisions')
                     )
                 )
 
             if not feed:
                 html_tmpl = '<span id="%s" style="display:none">, %s </span>'
             else:
                 html_tmpl = '<span id="%s"> %s </span>'
 
             morelinks = ', '.join(
               [lnk(rev, repo_name) for rev in revs[revs_limit:]]
             )
 
             if len(revs_ids) > revs_top_limit:
                 morelinks += ', ...'
 
             cs_links.append(html_tmpl % (uniq_id, morelinks))
         if len(revs) > 1:
             cs_links.append(compare_view)
         return ''.join(cs_links)
 
     def get_fork_name():
         repo_name = action_params
         url_ = url('summary_home', repo_name=repo_name)
         return _('Fork name %s') % link_to(action_params, url_)
 
     def get_user_name():
         user_name = action_params
         return user_name
 
     def get_users_group():
         group_name = action_params
         return group_name
 
     def get_pull_request():
         pull_request_id = action_params
         nice_id = db.PullRequest.make_nice_id(pull_request_id)
 
         deleted = user_log.repository is None
         if deleted:
             repo_name = user_log.repository_name
         else:
             repo_name = user_log.repository.repo_name
 
         return link_to(_('Pull request %s') % nice_id,
                     url('pullrequest_show', repo_name=repo_name,
                     pull_request_id=pull_request_id))
 
     def get_archive_name():
         archive_name = action_params
         return archive_name
 
     # action : translated str, callback(extractor), icon
     action_map = {
         'user_deleted_repo':           (_('[deleted] repository'),
                                         None, 'icon-trashcan'),
         'user_created_repo':           (_('[created] repository'),
                                         None, 'icon-plus'),
         'user_created_fork':           (_('[created] repository as fork'),
                                         None, 'icon-fork'),
         'user_forked_repo':            (_('[forked] repository'),
                                         get_fork_name, 'icon-fork'),
         'user_updated_repo':           (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'user_downloaded_archive':      (_('[downloaded] archive from repository'),
                                         get_archive_name, 'icon-download-cloud'),
         'admin_deleted_repo':          (_('[delete] repository'),
                                         None, 'icon-trashcan'),
         'admin_created_repo':          (_('[created] repository'),
                                         None, 'icon-plus'),
         'admin_forked_repo':           (_('[forked] repository'),
                                         None, 'icon-fork'),
         'admin_updated_repo':          (_('[updated] repository'),
                                         None, 'icon-pencil'),
         'admin_created_user':          (_('[created] user'),
                                         get_user_name, 'icon-user'),
         'admin_updated_user':          (_('[updated] user'),
                                         get_user_name, 'icon-user'),
         'admin_created_users_group':   (_('[created] user group'),
                                         get_users_group, 'icon-pencil'),
         'admin_updated_users_group':   (_('[updated] user group'),
                                         get_users_group, 'icon-pencil'),
         'user_commented_revision':     (_('[commented] on revision in repository'),
                                         get_cs_links, 'icon-comment'),
         'user_commented_pull_request': (_('[commented] on pull request for'),
                                         get_pull_request, 'icon-comment'),
         'user_closed_pull_request':    (_('[closed] pull request for'),
                                         get_pull_request, 'icon-ok'),
         'push':                        (_('[pushed] into'),
                                         get_cs_links, 'icon-move-up'),
         'push_local':                  (_('[committed via Kallithea] into repository'),
                                         get_cs_links, 'icon-pencil'),
         'push_remote':                 (_('[pulled from remote] into repository'),
                                         get_cs_links, 'icon-move-up'),
         'pull':                        (_('[pulled] from'),
                                         None, 'icon-move-down'),
         'started_following_repo':      (_('[started following] repository'),
                                         None, 'icon-heart'),
         'stopped_following_repo':      (_('[stopped following] repository'),
                                         None, 'icon-heart-empty'),
     }
 
     action_str = action_map.get(action, action)
     if feed:
         action = action_str[0].replace('[', '').replace(']', '')
     else:
         action = action_str[0] \
             .replace('[', '<b>') \
             .replace(']', '</b>')
 
     action_params_func = action_str[1] if callable(action_str[1]) else (lambda: "")
 
     def action_parser_icon():
         action = user_log.action
         action_params = None
         x = action.split(':')
 
         if len(x) > 1:
             action, action_params = x
 
         ico = action_map.get(action, ['', '', ''])[2]
         html = """<i class="%s"></i>""" % ico
         return literal(html)
 
     # returned callbacks we need to call to get
     return [lambda: literal(action), action_params_func, action_parser_icon]
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 def gravatar_div(email_address, cls='', size=30, **div_attributes):
     """Return an html literal with a span around a gravatar if they are enabled.
     Extra keyword parameters starting with 'div_' will get the prefix removed
     and '_' changed to '-' and be used as attributes on the div. The default
     class is 'gravatar'.
     """
-    from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
     if 'div_class' not in div_attributes:
         div_attributes['div_class'] = "gravatar"
     attributes = []
     for k, v in sorted(div_attributes.items()):
         assert k.startswith('div_'), k
         attributes.append(' %s="%s"' % (k[4:].replace('_', '-'), escape(v)))
     return literal("""<span%s>%s</span>""" %
                    (''.join(attributes),
                     gravatar(email_address, cls=cls, size=size)))
 
 
 def gravatar(email_address, cls='', size=30):
     """return html element of the gravatar
 
     This method will return an <img> with the resolution double the size (for
     retina screens) of the image. If the url returned from gravatar_url is
     empty then we fallback to using an icon.
 
     """
-    from tg import tmpl_context as c
     if not c.visual.use_gravatar:
         return ''
 
     src = gravatar_url(email_address, size * 2)
 
     if src:
         # here it makes sense to use style="width: ..." (instead of, say, a
         # stylesheet) because we using this to generate a high-res (retina) size
         html = ('<i class="icon-gravatar {cls}"'
                 ' style="font-size: {size}px;background-size: {size}px;background-image: url(\'{src}\')"'
                 '></i>').format(cls=cls, size=size, src=src)
 
     else:
         # if src is empty then there was no gravatar, so we use a font icon
         html = ("""<i class="icon-user {cls}" style="font-size: {size}px;"></i>"""
             .format(cls=cls, size=size))
 
     return literal(html)
 
 
 def gravatar_url(email_address, size=30, default=''):
-    from tg import tmpl_context as c
-
     if not c.visual.use_gravatar:
         return ""
 
     _def = '[e-mail]'  # default gravatar
     email_address = email_address or _def
 
     if email_address == _def:
         return default
 
     parsed_url = urllib.parse.urlparse(url.current(qualified=True))
     return (c.visual.gravatar_url or db.User.DEFAULT_GRAVATAR_URL) \
                .replace('{email}', email_address) \
                .replace('{md5email}', hashlib.md5(safe_bytes(email_address).lower()).hexdigest()) \
                .replace('{netloc}', parsed_url.netloc) \
                .replace('{scheme}', parsed_url.scheme) \
                .replace('{size}', str(size))
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in changeset page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([x.path
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No files')
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
 
     a, d = stats['added'], stats['deleted']
     width = 100
 
     if stats['binary']:
         # binary mode
         lbl = ''
         bin_op = 1
 
         if BIN_FILENODE in stats['ops']:
             lbl = 'bin+'
 
         if NEW_FILENODE in stats['ops']:
             lbl += _('new file')
             bin_op = NEW_FILENODE
         elif MOD_FILENODE in stats['ops']:
             lbl += _('mod')
             bin_op = MOD_FILENODE
         elif DEL_FILENODE in stats['ops']:
             lbl += _('del')
             bin_op = DEL_FILENODE
         elif RENAMED_FILENODE in stats['ops']:
             lbl += _('rename')
             bin_op = RENAMED_FILENODE
 
         # chmod can go with other operations
         if CHMOD_FILENODE in stats['ops']:
             _org_lbl = _('chmod')
             lbl += _org_lbl if lbl.endswith('+') else '+%s' % _org_lbl
 
         #import ipdb;ipdb.set_trace()
         b_d = '<div class="bin bin%s progress-bar" style="width:100%%">%s</div>' % (bin_op, lbl)
         b_a = '<div class="bin bin1" style="width:0%"></div>'
         return literal('<div style="width:%spx" class="progress">%s%s</div>' % (width, b_a, b_d))
 
     t = stats['added'] + stats['deleted']
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         # adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     d_a = '<div class="added progress-bar" style="width:%s%%">%s</div>' % (
         a_p, a_v
     )
     d_d = '<div class="deleted progress-bar" style="width:%s%%">%s</div>' % (
         d_p, d_v
     )
     return literal('<div class="progress" style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 _URLIFY_RE = re.compile(r'''
 # URL markup
 (?P<url>%s) |
 # @mention markup
 (?P<mention>%s) |
 # Changeset hash markup
 (?<!\w|[-_])
   (?P<hash>[0-9a-f]{12,40})
 (?!\w|[-_]) |
 # Markup of *bold text*
 (?:
   (?:^|(?<=\s))
   (?P<bold> [*] (?!\s) [^*\n]* (?<!\s) [*] )
   (?![*\w])
 ) |
 # "Stylize" markup
 \[see\ \=&gt;\ *(?P<seen>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[license\ \=&gt;\ *(?P<license>[a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\] |
 \[(?P<tagtype>requires|recommends|conflicts|base)\ \=&gt;\ *(?P<tagvalue>[a-zA-Z0-9\-\/]*)\] |
 \[(?:lang|language)\ \=&gt;\ *(?P<lang>[a-zA-Z\-\/\#\+]*)\] |
 \[(?P<tag>[a-z]+)\]
 ''' % (url_re.pattern, MENTIONS_REGEX.pattern),
     re.VERBOSE | re.MULTILINE | re.IGNORECASE)
 
 
 def urlify_text(s, repo_name=None, link_=None, truncate=None, stylize=False, truncatef=truncate):
     """
     Parses given text message and make literal html with markup.
     The text will be truncated to the specified length.
     Hashes are turned into changeset links to specified repository.
     URLs links to what they say.
     Issues are linked to given issue-server.
     If link_ is provided, all text not already linking somewhere will link there.
     >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* <b>markup/b>")
     literal('Urlify <a href="http://example.com/">http://example.com/</a> and &#39;<a href="https://example.com&apos">https://example.com&apos</a>; <b>*and*</b> &lt;b&gt;markup/b&gt;')
     """
 
     def _replace(match_obj):
         match_url = match_obj.group('url')
         if match_url is not None:
             return '<a href="%(url)s">%(url)s</a>' % {'url': match_url}
         mention = match_obj.group('mention')
         if mention is not None:
             return '<b>%s</b>' % mention
         hash_ = match_obj.group('hash')
         if hash_ is not None and repo_name is not None:
             return '<a class="changeset_hash" href="%(url)s">%(hash)s</a>' % {
                  'url': url('changeset_home', repo_name=repo_name, revision=hash_),
                  'hash': hash_,
                 }
         bold = match_obj.group('bold')
         if bold is not None:
             return '<b>*%s*</b>' % _urlify(bold[1:-1])
         if stylize:
             seen = match_obj.group('seen')
             if seen:
                 return '<div class="label label-meta" data-tag="see">see =&gt; %s</div>' % seen
             license = match_obj.group('license')
             if license:
                 return '<div class="label label-meta" data-tag="license"><a href="http://www.opensource.org/licenses/%s">%s</a></div>' % (license, license)
             tagtype = match_obj.group('tagtype')
             if tagtype:
                 tagvalue = match_obj.group('tagvalue')
                 return '<div class="label label-meta" data-tag="%s">%s =&gt; <a href="/%s">%s</a></div>' % (tagtype, tagtype, tagvalue, tagvalue)
             lang = match_obj.group('lang')
             if lang:
                 return '<div class="label label-meta" data-tag="lang">%s</div>' % lang
             tag = match_obj.group('tag')
             if tag:
                 return '<div class="label label-meta" data-tag="%s">%s</div>' % (tag, tag)
         return match_obj.group(0)
 
     def _urlify(s):
         """
         Extract urls from text and make html links out of them
         """
         return _URLIFY_RE.sub(_replace, s)
 
     if truncate is None:
         s = s.rstrip()
     else:
         s = truncatef(s, truncate, whole_word=True)
     s = html_escape(s)
     s = _urlify(s)
     if repo_name is not None:
         s = urlify_issues(s, repo_name)
     if link_ is not None:
         # make href around everything that isn't a href already
         s = linkify_others(s, link_)
     s = s.replace('\r\n', '<br/>').replace('\n', '<br/>')
     # Turn HTML5 into more valid HTML4 as required by some mail readers.
     # (This is not done in one step in html_escape, because character codes like
     # &#123; risk to be seen as an issue reference due to the presence of '#'.)
     s = s.replace("&apos;", "&#39;")
     return literal(s)
 
 
 def linkify_others(t, l):
     """Add a default link to html with links.
     HTML doesn't allow nesting of links, so the outer link must be broken up
     in pieces and give space for other links.
     """
     urls = re.compile(r'(\<a.*?\<\/a\>)',)
     links = []
     for e in urls.split(t):
         if e.strip() and not urls.match(e):
             links.append('<a class="message-link" href="%s">%s</a>' % (l, e))
         else:
             links.append(e)
 
     return ''.join(links)
 
 
 # Global variable that will hold the actual urlify_issues function body.
 # Will be set on first use when the global configuration has been read.
 _urlify_issues_f = None
 
 
 def urlify_issues(newtext, repo_name):
     """Urlify issue references according to .ini configuration"""
     global _urlify_issues_f
     if _urlify_issues_f is None:
         assert kallithea.CONFIG['sqlalchemy.url'] # make sure config has been loaded
 
         # Build chain of urlify functions, starting with not doing any transformation
         def tmp_urlify_issues_f(s):
             return s
 
         issue_pat_re = re.compile(r'issue_pat(.*)')
         for k in kallithea.CONFIG:
             # Find all issue_pat* settings that also have corresponding server_link and prefix configuration
             m = issue_pat_re.match(k)
             if m is None:
                 continue
             suffix = m.group(1)
             issue_pat = kallithea.CONFIG.get(k)
             issue_server_link = kallithea.CONFIG.get('issue_server_link%s' % suffix)
             issue_sub = kallithea.CONFIG.get('issue_sub%s' % suffix)
             issue_prefix = kallithea.CONFIG.get('issue_prefix%s' % suffix)
             if issue_prefix:
                 log.error('found unsupported issue_prefix%s = %r - use issue_sub%s instead', suffix, issue_prefix, suffix)
             if not issue_pat:
                 log.error('skipping incomplete issue pattern %r: it needs a regexp', k)
                 continue
             if not issue_server_link:
                 log.error('skipping incomplete issue pattern %r: it needs issue_server_link%s', k, suffix)
                 continue
             if issue_sub is None: # issue_sub can be empty but should be present
                 log.error('skipping incomplete issue pattern %r: it needs (a potentially empty) issue_sub%s', k, suffix)
                 continue
 
             # Wrap tmp_urlify_issues_f with substitution of this pattern, while making sure all loop variables (and compiled regexpes) are bound
             try:
                 issue_re = re.compile(issue_pat)
             except re.error as e:
                 log.error('skipping invalid issue pattern %r: %r -> %r %r. Error: %s', k, issue_pat, issue_server_link, issue_sub, str(e))
                 continue
 
             log.debug('issue pattern %r: %r -> %r %r', k, issue_pat, issue_server_link, issue_sub)
 
             def issues_replace(match_obj,
                                issue_server_link=issue_server_link, issue_sub=issue_sub):
                 try:
                     issue_url = match_obj.expand(issue_server_link)
                 except (IndexError, re.error) as e:
                     log.error('invalid issue_url setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                     issue_url = issue_server_link
                 issue_url = issue_url.replace('{repo}', repo_name)
                 issue_url = issue_url.replace('{repo_name}', repo_name.split(kallithea.URL_SEP)[-1])
                 # if issue_sub is empty use the matched issue reference verbatim
                 if not issue_sub:
                     issue_text = match_obj.group()
                 else:
                     try:
                         issue_text = match_obj.expand(issue_sub)
                     except (IndexError, re.error) as e:
                         log.error('invalid issue_sub setting %r -> %r %r. Error: %s', issue_pat, issue_server_link, issue_sub, str(e))
                         issue_text = match_obj.group()
 
                 return (
                     '<a class="issue-tracker-link" href="%(url)s">'
                     '%(text)s'
                     '</a>'
                     ) % {
                      'url': issue_url,
                      'text': issue_text,
                     }
 
             def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f):
                 return issue_re.sub(issues_replace, chain_f(s))
 
         # Set tmp function globally - atomically
         _urlify_issues_f = tmp_urlify_issues_f
 
     return _urlify_issues_f(newtext)
 
 
 def render_w_mentions(source, repo_name=None):
     """
     Render plain text with revision hashes and issue references urlified
     and with @mention highlighting.
     """
     s = safe_str(source)
     s = urlify_text(s, repo_name=repo_name)
     return literal('<div class="formatted-fixed">%s</div>' % s)
 
 
 def changeset_status(repo, revision):
     return ChangesetStatusModel().get_status(repo, revision)
 
 
 def changeset_status_lbl(changeset_status):
     return db.ChangesetStatus.get_status_lbl(changeset_status)
 
 
 def get_permission_name(key):
     return dict(db.Permission.PERMS).get(key)
 
 
 def journal_filter_help():
     return _(textwrap.dedent('''
         Example filter terms:
             repository:vcs
             username:developer
             action:*push*
             ip:127.0.0.1
             date:20120101
             date:[20120101100000 TO 20120102]
 
         Generate wildcards using '*' character:
             "repository:vcs*" - search everything starting with 'vcs'
             "repository:*vcs*" - search for repository containing 'vcs'
 
         Optional AND / OR operators in queries
             "repository:vcs OR repository:test"
             "username:test AND repository:test*"
     '''))
 
 
 def ip_range(ip_addr):
     s, e = db.UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)

kallithea/lib/hooks.py

@@ -1,406 +1,405 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.hooks
 ~~~~~~~~~~~~~~~~~~~
 
 Hooks run by Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import os
 import sys
 import time
 
 import mercurial.scmutil
+import paste.deploy
 
 import kallithea
 from kallithea.lib import webutils
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import action_logger, make_ui
 from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.model import db
 
 
 def _get_scm_size(alias, root_path):
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, dirs, files in os.walk(root_path):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = webutils.format_byte_size(size_scm)
     size_root_f = webutils.format_byte_size(size_root)
     size_total_f = webutils.format_byte_size(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Show size of Mercurial repository.
 
     Called as Mercurial hook changegroup.repo_size after push.
     """
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', safe_str(repo.root))
 
     last_cs = repo[len(repo) - 1]
 
     msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), ascii_str(last_cs.hex())[:12]
     )
     ui.status(safe_bytes(msg))
 
 
 def log_pull_action(ui, repo, **kwargs):
     """Logs user last pull action
 
     Called as Mercurial hook outgoing.pull_logger or from Kallithea before invoking Git.
 
     Does *not* use the action from the hook environment but is always 'pull'.
     """
     ex = get_hook_environment()
 
     user = db.User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     callback = getattr(kallithea.EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
 
 
 def log_push_action(ui, repo, node, node_last, **kwargs):
     """
     Register that changes have been added to the repo - log the action *and* invalidate caches.
     Note: This hook is not only logging, but also the side effect invalidating
     caches! The function should perhaps be renamed.
 
     Called as Mercurial hook changegroup.kallithea_log_push_action .
 
     The pushed changesets is given by the revset 'node:node_last'.
     """
     revs = [ascii_str(repo[r].hex()) for r in mercurial.scmutil.revrange(repo, [b'%s:%s' % (node, node_last)])]
     process_pushed_raw_ids(revs)
 
 
 def process_pushed_raw_ids(revs):
     """
     Register that changes have been added to the repo - log the action *and* invalidate caches.
 
     Called from Mercurial changegroup.kallithea_log_push_action calling hook log_push_action,
     or from the Git post-receive hook calling handle_git_post_receive ...
     or from scm _handle_push.
     """
     ex = get_hook_environment()
 
     action = '%s:%s' % (ex.action, ','.join(revs))
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
 
     from kallithea.model.scm import ScmModel
     ScmModel().mark_for_invalidation(ex.repository)
 
     # extension hook call
     callback = getattr(kallithea.EXTENSIONS, 'PUSH_HOOK', None)
     if callable(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
 
 
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook.
 
     :param repository: dict dump of repository object
     :param created_by: username who created repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     callback = getattr(kallithea.EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         callback(**kw)
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     callback = getattr(kallithea.EXTENSIONS, 'PRE_CREATE_USER_HOOK', None)
     if callable(callback):
         allowed, reason = callback(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 
 
 def log_create_user(user_dict, created_by, **kwargs):
     """
     Post create user Hook.
 
     :param user_dict: dict dump of user object
 
     available keys for user_dict:
 
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
 
     """
     callback = getattr(kallithea.EXTENSIONS, 'CREATE_USER_HOOK', None)
     if callable(callback):
         callback(created_by=created_by, **user_dict)
 
 
 def log_create_pullrequest(pullrequest_dict, created_by, **kwargs):
     """
     Post create pull request hook.
 
     :param pullrequest_dict: dict dump of pull request object
     """
     callback = getattr(kallithea.EXTENSIONS, 'CREATE_PULLREQUEST_HOOK', None)
     if callable(callback):
         return callback(created_by=created_by, **pullrequest_dict)
 
     return 0
 
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
     Post delete repository Hook.
 
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     callback = getattr(kallithea.EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         callback(**kw)
 
 
 def log_delete_user(user_dict, deleted_by, **kwargs):
     """
     Post delete user Hook.
 
     :param user_dict: dict dump of user object
 
     available keys for user_dict:
 
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
 
     """
     callback = getattr(kallithea.EXTENSIONS, 'DELETE_USER_HOOK', None)
     if callable(callback):
         callback(deleted_by=deleted_by, **user_dict)
 
 
 def _hook_environment(repo_path):
     """
     Create a light-weight environment for stand-alone scripts and return an UI and the
     db repository.
 
     Git hooks are executed as subprocess of Git while Kallithea is waiting, and
     they thus need enough info to be able to create an app environment and
     connect to the database.
     """
-    import paste.deploy
-
     import kallithea.config.application
 
     extras = get_hook_environment()
 
     path_to_ini_file = extras['config']
     config = paste.deploy.appconfig('config:' + path_to_ini_file)
     #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
     kallithea.config.application.make_app(config.global_conf, **config.local_conf)
 
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
 
     repo = db.Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database' % repo_path)
 
     baseui = make_ui()
     return baseui, repo
 
 
 def handle_git_pre_receive(repo_path, git_stdin_lines):
     """Called from Git pre-receive hook.
     The returned value is used as hook exit code and must be 0.
     """
     # Currently unused. TODO: remove?
     return 0
 
 
 def handle_git_post_receive(repo_path, git_stdin_lines):
     """Called from Git post-receive hook.
     The returned value is used as hook exit code and must be 0.
     """
     try:
         baseui, repo = _hook_environment(repo_path)
     except HookEnvironmentError as e:
         sys.stderr.write("Skipping Kallithea Git post-recieve hook %r.\nGit was apparently not invoked by Kallithea: %s\n" % (sys.argv[0], e))
         return 0
 
     # the post push hook should never use the cached instance
     scm_repo = repo.scm_instance_no_cache()
 
     rev_data = []
     for l in git_stdin_lines:
         old_rev, new_rev, ref = l.strip().split(' ')
         _ref_data = ref.split('/')
         if _ref_data[1] in ['tags', 'heads']:
             rev_data.append({'old_rev': old_rev,
                              'new_rev': new_rev,
                              'ref': ref,
                              'type': _ref_data[1],
                              'name': '/'.join(_ref_data[2:])})
 
     git_revs = []
     for push_ref in rev_data:
         _type = push_ref['type']
         if _type == 'heads':
             if push_ref['old_rev'] == EmptyChangeset().raw_id:
                 # update the symbolic ref if we push new repo
                 if scm_repo.is_empty():
                     scm_repo._repo.refs.set_symbolic_ref(
                         b'HEAD',
                         b'refs/heads/%s' % safe_bytes(push_ref['name']))
 
                 # build exclude list without the ref
                 cmd = ['for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout = scm_repo.run_git_command(cmd)
                 ref = push_ref['ref']
                 heads = [head for head in stdout.splitlines() if head != ref]
                 # now list the git revs while excluding from the list
                 cmd = ['log', push_ref['new_rev'], '--reverse', '--pretty=format:%H']
                 cmd.append('--not')
                 cmd.extend(heads) # empty list is ok
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
 
             elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                 # delete branch case
                 git_revs += ['delete_branch=>%s' % push_ref['name']]
             else:
                 cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
                        '--reverse', '--pretty=format:%H']
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
 
         elif _type == 'tags':
             git_revs += ['tag=>%s' % push_ref['name']]
 
     process_pushed_raw_ids(git_revs)
 
     return 0
 
 
 # Almost exactly like Mercurial contrib/hg-ssh:
 def rejectpush(ui, **kwargs):
     """Mercurial hook to be installed as pretxnopen and prepushkey for read-only repos.
     Return value 1 will make the hook fail and reject the push.
     """
     ex = get_hook_environment()
     ui.warn(safe_bytes("Push access to %r denied\n" % ex.repository))
     return 1

kallithea/lib/utils2.py

@@ -1,551 +1,551 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils2
 ~~~~~~~~~~~~~~~~~~~~
 
 Some simple helper functions.
 Note: all these functions should be independent of Kallithea classes, i.e.
 models, controllers, etc.  to prevent import cycles.
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 5, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import binascii
 import datetime
 import json
 import os
 import re
 import time
 import urllib.parse
 
 import urlobject
 from dateutil import relativedelta
 from sqlalchemy.engine import url as sa_url
 from sqlalchemy.exc import ArgumentError
+from tg import tmpl_context
 from tg.i18n import ugettext as _
 from tg.i18n import ungettext
 from tg.support.converters import asbool, aslist
 from webhelpers2.text import collapse, remove_formatting, strip_tags
 
 import kallithea
 from kallithea.lib import webutils
 from kallithea.lib.vcs.backends.base import BaseRepository, EmptyChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str  # re-export
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 
 
 try:
     import pwd
 except ImportError:
     pass
 
 
 # mute pyflakes "imported but unused"
 assert asbool
 assert aslist
 assert ascii_bytes
 assert ascii_str
 assert safe_bytes
 assert safe_str
 assert LazyProperty
 
 
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" according to given mode
 
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
 
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     if mode == 0:
         line = line.replace('\r\n', '\n')
         line = line.replace('\r', '\n')
     elif mode == 1:
         line = line.replace('\r\n', '\r')
         line = line.replace('\n', '\r')
     elif mode == 2:
         line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 
 
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def generate_api_key():
     """
     Generates a random (presumably unique) API key.
 
     This value is used in URLs and "Bearer" HTTP Authorization headers,
     which in practice means it should only contain URL-safe characters
     (RFC 3986):
 
         unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
     """
     # Hexadecimal certainly qualifies as URL-safe.
     return ascii_str(binascii.hexlify(os.urandom(20)))
 
 
 def safe_int(val, default=None):
     """
     Returns int() of val if val is not convertable to int use default
     instead
 
     :param val:
     :param default:
     """
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
     return val
 
 
 def remove_suffix(s, suffix):
     if s.endswith(suffix):
         s = s[:-1 * len(suffix)]
     return s
 
 
 def remove_prefix(s, prefix):
     if s.startswith(prefix):
         s = s[len(prefix):]
     return s
 
 
 def shorter(s, size=20, firstline=False, postfix='...'):
     """Truncate s to size, including the postfix string if truncating.
     If firstline, truncate at newline.
     """
     if firstline:
         s = s.split('\n', 1)[0].rstrip()
     if len(s) > size:
         return s[:size - len(postfix)] + postfix
     return s
 
 
 def age(prevdate, show_short_version=False, now=None):
     """
     turns a datetime into an age string.
     If show_short_version is True, then it will generate a not so accurate but shorter string,
     example: 2days ago, instead of 2 days and 23 hours ago.
 
     :param prevdate: datetime object
     :param show_short_version: if it should approximate the date and return a shorter string
     :rtype: str
     :returns: str words describing age
     """
     now = now or datetime.datetime.now()
     order = ['year', 'month', 'day', 'hour', 'minute', 'second']
     deltas = {}
     future = False
 
     if prevdate > now:
         now, prevdate = prevdate, now
         future = True
     if future:
         prevdate = prevdate.replace(microsecond=0)
     # Get date parts deltas
     for part in order:
         d = relativedelta.relativedelta(now, prevdate)
         deltas[part] = getattr(d, part + 's')
 
     # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
     # not 1 hour, -59 minutes and -59 seconds)
     for num, length in [(5, 60), (4, 60), (3, 24)]:  # seconds, minutes, hours
         part = order[num]
         carry_part = order[num - 1]
 
         if deltas[part] < 0:
             deltas[part] += length
             deltas[carry_part] -= 1
 
     # Same thing for days except that the increment depends on the (variable)
     # number of days in the month
     month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
     if deltas['day'] < 0:
         if prevdate.month == 2 and (prevdate.year % 4 == 0 and
             (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)
         ):
             deltas['day'] += 29
         else:
             deltas['day'] += month_lengths[prevdate.month - 1]
 
         deltas['month'] -= 1
 
     if deltas['month'] < 0:
         deltas['month'] += 12
         deltas['year'] -= 1
 
     # In short version, we want nicer handling of ages of more than a year
     if show_short_version:
         if deltas['year'] == 1:
             # ages between 1 and 2 years: show as months
             deltas['month'] += 12
             deltas['year'] = 0
         if deltas['year'] >= 2:
             # ages 2+ years: round
             if deltas['month'] > 6:
                 deltas['year'] += 1
                 deltas['month'] = 0
 
     # Format the result
     fmt_funcs = {
         'year': lambda d: ungettext('%d year', '%d years', d) % d,
         'month': lambda d: ungettext('%d month', '%d months', d) % d,
         'day': lambda d: ungettext('%d day', '%d days', d) % d,
         'hour': lambda d: ungettext('%d hour', '%d hours', d) % d,
         'minute': lambda d: ungettext('%d minute', '%d minutes', d) % d,
         'second': lambda d: ungettext('%d second', '%d seconds', d) % d,
     }
 
     for i, part in enumerate(order):
         value = deltas[part]
         if value == 0:
             continue
 
         if i < 5:
             sub_part = order[i + 1]
             sub_value = deltas[sub_part]
         else:
             sub_value = 0
 
         if sub_value == 0 or show_short_version:
             if future:
                 return _('in %s') % fmt_funcs[part](value)
             else:
                 return _('%s ago') % fmt_funcs[part](value)
         if future:
             return _('in %s and %s') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
         else:
             return _('%s and %s ago') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
 
     return _('just now')
 
 
 def fmt_date(date):
     if date:
         return date.strftime("%Y-%m-%d %H:%M:%S")
     return ""
 
 
 def uri_filter(uri):
     """
     Removes user:password from given url string
 
     :param uri:
     :rtype: str
     :returns: filtered list of strings
     """
     if not uri:
         return []
 
     proto = ''
 
     for pat in ('https://', 'http://', 'git://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
 
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
 
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
 
     return [_f for _f in [proto, host, port] if _f]
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
 
     :param uri:
     """
 
     uri = uri_filter(uri)
     # check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
 
     return ''.join(uri)
 
 
 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
     parsed_url = urlobject.URLObject(prefix_url)
     prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))
     try:
         system_user = pwd.getpwuid(os.getuid()).pw_name
     except NameError: # TODO: support all systems - especially Windows
         system_user = 'kallithea' # hardcoded default value ...
     args = {
         'scheme': parsed_url.scheme,
         'user': urllib.parse.quote(username or ''),
         'netloc': parsed_url.netloc + prefix,  # like "hostname:port/prefix" (with optional ":port" and "/prefix")
         'prefix': prefix, # undocumented, empty or starting with /
         'repo': repo_name,
         'repoid': str(repo_id),
         'system_user': system_user,
         'hostname': parsed_url.hostname,
     }
     url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
 
     # remove leading @ sign if it's present. Case of empty user
     url_obj = urlobject.URLObject(url)
     if not url_obj.username:
         url_obj = url_obj.with_username(None)
 
     return str(url_obj)
 
 
 def short_ref_name(ref_type, ref_name):
     """Return short description of PR ref - revs will be truncated"""
     if ref_type == 'rev':
         return ref_name[:12]
     return ref_name
 
 
 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
     """
     Return full markup for a PR ref to changeset_home for a changeset.
     If ref_type is 'branch', it will link to changelog.
     ref_name is shortened if ref_type is 'rev'.
     if rev is specified, show it too, explicitly linking to that revision.
     """
     txt = short_ref_name(ref_type, ref_name)
     if ref_type == 'branch':
         u = webutils.url('changelog_home', repo_name=repo_name, branch=ref_name)
     else:
         u = webutils.url('changeset_home', repo_name=repo_name, revision=ref_name)
     l = webutils.link_to(repo_name + '#' + txt, u)
     if rev and ref_type != 'rev':
         l = webutils.literal('%s (%s)' % (l, webutils.link_to(rev[:12], webutils.url('changeset_home', repo_name=repo_name, revision=rev))))
     return l
 
 
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a
     repo it returns a Dummy one instead
 
     :param repo:
     :param rev:
     """
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s' % type(repo))
 
     try:
         cs = repo.get_changeset(rev)
     except (RepositoryError, LookupError):
         cs = EmptyChangeset(requested_revision=rev)
     return cs
 
 
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 
 
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 
 
 # Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()
 # Check char before @ - it must not look like we are in an email addresses.
 # Matching is greedy so we don't have to look beyond the end.
 MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')
 
 
 def extract_mentioned_usernames(text):
     r"""
     Returns list of (possible) usernames @mentioned in given text.
 
     >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')
     ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz']
     """
     return MENTIONS_REGEX.findall(text)
 
 
 class AttributeDict(dict):
     def __getattr__(self, attr):
         return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
 def obfuscate_url_pw(engine):
     try:
         _url = sa_url.make_url(engine or '')
     except ArgumentError:
         return engine
     if _url.password:
         _url.password = 'XXXXX'
     return str(_url)
 
 
 class HookEnvironmentError(Exception): pass
 
 
 def get_hook_environment():
     """
     Get hook context by deserializing the global KALLITHEA_EXTRAS environment
     variable.
 
     Called early in Git out-of-process hooks to get .ini config path so the
     basic environment can be configured properly. Also used in all hooks to get
     information about the action that triggered it.
     """
 
     try:
         kallithea_extras = os.environ['KALLITHEA_EXTRAS']
     except KeyError:
         raise HookEnvironmentError("Environment variable KALLITHEA_EXTRAS not found")
 
     extras = json.loads(kallithea_extras)
     for k in ['username', 'repository', 'scm', 'action', 'ip', 'config']:
         try:
             extras[k]
         except KeyError:
             raise HookEnvironmentError('Missing key %s in KALLITHEA_EXTRAS %s' % (k, extras))
 
     return AttributeDict(extras)
 
 
 def set_hook_environment(username, ip_addr, repo_name, repo_alias, action=None):
     """Prepare global context for running hooks by serializing data in the
     global KALLITHEA_EXTRAS environment variable.
 
     Most importantly, this allow Git hooks to do proper logging and updating of
     caches after pushes.
 
     Must always be called before anything with hooks are invoked.
     """
     extras = {
         'ip': ip_addr, # used in log_push/pull_action action_logger
         'username': username,
         'action': action or 'push_local', # used in log_push_action_raw_ids action_logger
         'repository': repo_name,
         'scm': repo_alias, # used to pick hack in log_push_action_raw_ids
         'config': kallithea.CONFIG['__file__'], # used by git hook to read config
     }
     os.environ['KALLITHEA_EXTRAS'] = json.dumps(extras)
 
 
 def get_current_authuser():
     """
     Gets kallithea user from threadlocal tmpl_context variable if it's
     defined, else returns None.
     """
-    from tg import tmpl_context
     try:
         return getattr(tmpl_context, 'authuser', None)
     except TypeError:  # No object (name: context) has been registered for this thread
         return None
 
 
 def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):
     return _cleanstringsub('_', s).rstrip('_')
 
 
 def recursive_replace(str_, replace=' '):
     """
     Recursive replace of given sign to just one instance
 
     :param str_: given string
     :param replace: char to find and replace multiple instances
 
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
 
     if str_.find(replace * 2) == -1:
         return str_
     else:
         str_ = str_.replace(replace * 2, replace)
         return recursive_replace(str_, replace)
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     slug = remove_formatting(value)
     slug = strip_tags(slug)
 
     for c in r"""`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 
 
 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
     while True:
         ok = input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print(complaint)

kallithea/lib/vcs/backends/git/inmemory.py

@@ -1,196 +1,197 b''
 import datetime
 import posixpath
 import stat
 import time
 
 from dulwich import objects
 
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import ascii_str, safe_bytes
 
+from . import repository
+
 
 class GitInMemoryChangeset(BaseInMemoryChangeset):
 
     def commit(self, message, author, parents=None, branch=None, date=None,
                **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         self.check_integrity(parents)
 
-        from .repository import GitRepository
         if branch is None:
-            branch = GitRepository.DEFAULT_BRANCH_NAME
+            branch = repository.GitRepository.DEFAULT_BRANCH_NAME
 
         repo = self.repository._repo
         object_store = repo.object_store
 
         ENCODING = b"UTF-8"  # TODO: should probably be kept in sync with safe_str/safe_bytes and vcs/conf/settings.py DEFAULT_ENCODINGS
 
         # Create tree and populates it with blobs
         commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or \
             objects.Tree()
         for node in self.added + self.changed:
             # Compute subdirs if needed
             dirpath, nodename = posixpath.split(node.path)
             dirnames = safe_bytes(dirpath).split(b'/') if dirpath else []
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = self.repository._repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create subtrees
             # for dirnames (in reverse order) [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node.content)
 
             node_path = safe_bytes(node.name)
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node_path] = node.mode, blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     #newtree.add(stat.S_IFDIR, dirname, curtree.id)
                     newtree[dirname] = stat.S_IFDIR, curtree.id
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = stat.S_IFDIR, curtree.id
             else:
                 parent.add(name=node_path, mode=node.mode, hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in
                 zip(ancestors, ancestors[1:])]
             ):
                 parent[path] = stat.S_IFDIR, tree.id
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
         for node in self.removed:
             paths = safe_bytes(node.path).split(b'/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = self.repository._repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(list(zip(paths, trees))):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         commit.parents = [p._commit.id for p in self.parents if p]
         commit.author = commit.committer = safe_bytes(author)
         commit.encoding = ENCODING
         commit.message = safe_bytes(message)
 
         # Compute date
         if date is None:
             date = time.time()
         elif isinstance(date, datetime.datetime):
             date = time.mktime(date.timetuple())
 
         author_time = kwargs.pop('author_time', date)
         commit.commit_time = int(date)
         commit.author_time = int(author_time)
         tz = time.timezone
         author_tz = kwargs.pop('author_timezone', tz)
         commit.commit_timezone = tz
         commit.author_timezone = author_tz
 
         object_store.add_object(commit)
 
         # Update vcs repository object & recreate dulwich repo
         ref = b'refs/heads/%s' % safe_bytes(branch)
         repo.refs[ref] = commit.id
         self.repository.revisions.append(ascii_str(commit.id))
         # invalidate parsed refs after commit
         self.repository._parsed_refs = self.repository._get_parsed_refs()
         tip = self.repository.get_changeset()
         self.reset()
         return tip
 
     def _get_missing_trees(self, path, root_tree):
         """
         Creates missing ``Tree`` objects for the given path.
 
         :param path: path given as a string. It may be a path to a file node
           (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must
           end with slash (i.e. ``foo/bar/``).
         :param root_tree: ``dulwich.objects.Tree`` object from which we start
           traversing (should be commit's root tree)
         """
         dirpath = posixpath.split(path)[0]
         dirs = dirpath.split('/')
         if not dirs or dirs == ['']:
             return []
 
         def get_tree_for_dir(tree, dirname):
             for name, mode, id in tree.items():
                 if name == dirname:
                     obj = self.repository._repo[id]
                     if isinstance(obj, objects.Tree):
                         return obj
                     else:
                         raise RepositoryError("Cannot create directory %s "
                             "at tree %s as path is occupied and is not a "
                             "Tree" % (dirname, tree))
             return None
 
         trees = []
         parent = root_tree
         for dirname in dirs:
             tree = get_tree_for_dir(parent, dirname)
             if tree is None:
                 tree = objects.Tree()
                 parent.add(stat.S_IFDIR, dirname, tree.id)
                 parent = tree
             # Always append tree
             trees.append(tree)
         return trees

kallithea/lib/vcs/backends/git/repository.py

@@ -1,778 +1,776 b''
 # -*- coding: utf-8 -*-
 """
     vcs.backends.git.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Git repository implementation.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import errno
 import logging
 import os
 import re
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 
 import mercurial.util  # import url as hg_url
 from dulwich.client import SubprocessGitClient
 from dulwich.config import ConfigFile
 from dulwich.objects import Tag
 from dulwich.repo import NotGitRepository, Repo
 from dulwich.server import update_server_info
 
 from kallithea.lib.vcs import subprocessio
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
                                           TagDoesNotExistError)
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, makedate, safe_bytes, safe_str
 from kallithea.lib.vcs.utils.helpers import get_urllib_request_handlers
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath, get_user_home
 
-from .changeset import GitChangeset
-from .inmemory import GitInMemoryChangeset
-from .workdir import GitWorkdir
+from . import changeset, inmemory, workdir
 
 
 SHA_PATTERN = re.compile(r'^([0-9a-fA-F]{12}|[0-9a-fA-F]{40})$')
 
 log = logging.getLogger(__name__)
 
 
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
     scm = 'git'
 
     def __init__(self, repo_path, create=False, src_url=None,
                  update_after_clone=False, bare=False):
 
         self.path = abspath(repo_path)
         self.repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = self.repo.bare
 
     @property
     def _config_files(self):
         return [
             self.bare and abspath(self.path, 'config')
                       or abspath(self.path, '.git', 'config'),
              abspath(get_user_home(), '.gitconfig'),
          ]
 
     @property
     def _repo(self):
         return self.repo
 
     @property
     def head(self):
         try:
             return self._repo.head()
         except KeyError:
             return None
 
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
 
         try:
             self.revisions[0]
         except (KeyError, IndexError):
             return True
         return False
 
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
 
     @classmethod
     def _run_git_command(cls, cmd, cwd=None):
         """
         Runs given ``cmd`` as git command and returns output bytes in a tuple
         (stdout, stderr) ... or raise RepositoryError.
 
         :param cmd: git command to be executed
         :param cwd: passed directly to subprocess
         """
         # need to clean fix GIT_DIR !
         gitenv = dict(os.environ)
         gitenv.pop('GIT_DIR', None)
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
 
         assert isinstance(cmd, list), cmd
         cmd = [settings.GIT_EXECUTABLE_PATH, '-c', 'core.quotepath=false'] + cmd
         try:
             p = subprocessio.SubprocessIOChunker(cmd, cwd=cwd, env=gitenv, shell=False)
         except (EnvironmentError, OSError) as err:
             # output from the failing process is in str(EnvironmentError)
             msg = ("Couldn't run git command %s.\n"
                    "Subprocess failed with '%s': %s\n" %
                    (cmd, type(err).__name__, err)
             ).strip()
             log.error(msg)
             raise RepositoryError(msg)
 
         try:
             stdout = b''.join(p.output)
             stderr = b''.join(p.error)
         finally:
             p.close()
         # TODO: introduce option to make commands fail if they have any stderr output?
         if stderr:
             log.debug('stderr from %s:\n%s', cmd, stderr)
         else:
             log.debug('stderr from %s: None', cmd)
         return stdout, stderr
 
     def run_git_command(self, cmd):
         """
         Runs given ``cmd`` as git command with cwd set to current repo.
         Returns stdout as unicode str ... or raise RepositoryError.
         """
         cwd = None
         if os.path.isdir(self.path):
             cwd = self.path
         stdout, _stderr = self._run_git_command(cmd, cwd=cwd)
         return safe_str(stdout)
 
     @classmethod
     def _check_url(cls, url):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         if url.startswith('git://'):
             return True
 
         if '+' in url[:url.find('://')]:
             url = url[url.find('+') + 1:]
 
         url_obj = mercurial.util.url(safe_bytes(url))
         test_uri, handlers = get_urllib_request_handlers(url_obj)
         if not test_uri.endswith(b'info/refs'):
             test_uri = test_uri.rstrip(b'/') + b'/info/refs'
 
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
 
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         req = urllib.request.Request(
             "%s?%s" % (
                 safe_str(test_uri),
                 urllib.parse.urlencode({"service": 'git-upload-pack'})
             ))
 
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if b'service=git-upload-pack' not in gitdata:
             raise urllib.error.URLError(
                 "url [%s] does not look like an git" % cleaned_uri)
 
         return True
 
     def _get_repo(self, create, src_url=None, update_after_clone=False,
                   bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError("Location already exist")
         if src_url and not create:
             raise RepositoryError("Create should be set to True if src_url is "
                                   "given (clone operation creates repository)")
         try:
             if create and src_url:
                 GitRepository._check_url(src_url)
                 self.clone(src_url, update_after_clone, bare)
                 return Repo(self.path)
             elif create:
                 os.makedirs(self.path)
                 if bare:
                     return Repo.init_bare(self.path)
                 else:
                     return Repo.init(self.path)
             else:
                 return Repo(self.path)
         except (NotGitRepository, OSError) as err:
             raise RepositoryError(err)
 
     def _get_all_revisions(self):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
         try:
             self._repo.head()
         except KeyError:
             return []
 
         rev_filter = settings.GIT_REV_FILTER
         cmd = ['rev-list', rev_filter, '--reverse', '--date-order']
         try:
             so = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return so.splitlines()
 
     def _get_all_revisions2(self):
         # alternate implementation using dulwich
         includes = [ascii_str(sha) for key, (sha, type_) in self._parsed_refs.items()
                     if type_ != b'T']
         return [c.commit.id for c in self._repo.get_walker(include=includes)]
 
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
 
         if revision in (None, '', 'tip', 'HEAD', 'head', -1):
             revision = -1
 
         if isinstance(revision, int):
             try:
                 return self.revisions[revision]
             except IndexError:
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
 
         if isinstance(revision, str):
             if revision.isdigit() and (len(revision) < 12 or len(revision) == revision.count('0')):
                 try:
                     return self.revisions[int(revision)]
                 except IndexError:
                     msg = "Revision %r does not exist for %s" % (revision, self)
                     raise ChangesetDoesNotExistError(msg)
 
             # get by branch/tag name
             _ref_revision = self._parsed_refs.get(safe_bytes(revision))
             if _ref_revision:  # and _ref_revision[1] in [b'H', b'RH', b'T']:
                 return ascii_str(_ref_revision[0])
 
             if revision in self.revisions:
                 return revision
 
             # maybe it's a tag ? we don't have them in self.revisions
             if revision in self.tags.values():
                 return revision
 
             if SHA_PATTERN.match(revision):
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
 
         raise ChangesetDoesNotExistError("Given revision %r not recognized" % revision)
 
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns ``GitChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return self._get_revision(ref_name)
 
     def _get_archives(self, archive_name='tip'):
 
         for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
             yield {"type": i[0], "extension": i[1], "node": archive_name}
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall to
         filesystem (``file:///``) schema.
         """
         if url != 'default' and '://' not in url:
             url = ':///'.join(('file', url))
         return url
 
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
 
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             idx_loc = '' if self.bare else '.git'
             # fallback to filesystem
             in_path = os.path.join(self.path, idx_loc, "index")
             he_path = os.path.join(self.path, idx_loc, "HEAD")
             if os.path.exists(in_path):
                 return os.stat(in_path).st_mtime
             else:
                 return os.stat(he_path).st_mtime
 
     @LazyProperty
     def description(self):
         return safe_str(self._repo.get_description() or b'unknown')
 
     @LazyProperty
     def contact(self):
         undefined_contact = 'Unknown'
         return undefined_contact
 
     @property
     def branches(self):
         if not self.revisions:
             return {}
         _branches = [(safe_str(key), ascii_str(sha))
                      for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']
         return OrderedDict(sorted(_branches, key=(lambda ctx: ctx[0]), reverse=False))
 
     @LazyProperty
     def closed_branches(self):
         return {}
 
     @LazyProperty
     def tags(self):
         return self._get_tags()
 
     def _get_tags(self):
         if not self.revisions:
             return {}
         _tags = [(safe_str(key), ascii_str(sha))
                  for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']
         return OrderedDict(sorted(_tags, key=(lambda ctx: ctx[0]), reverse=True))
 
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         message = message or "Added tag %s for commit %s" % (name,
             changeset.raw_id)
         self._repo.refs[b"refs/tags/%s" % safe_bytes(name)] = changeset._commit.id
 
         self._parsed_refs = self._get_parsed_refs()
         self.tags = self._get_tags()
         return changeset
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         # self._repo.refs is a DiskRefsContainer, and .path gives the full absolute path of '.git'
         tagpath = os.path.join(safe_str(self._repo.refs.path), 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._parsed_refs = self._get_parsed_refs()
             self.tags = self._get_tags()
         except OSError as e:
             raise RepositoryError(e.strerror)
 
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return {}
 
     @LazyProperty
     def _parsed_refs(self):
         return self._get_parsed_refs()
 
     def _get_parsed_refs(self):
         """Return refs as a dict, like:
         { b'v0.2.0': [b'599ba911aa24d2981225f3966eb659dfae9e9f30', b'T'] }
         """
         _repo = self._repo
         refs = _repo.get_refs()
         keys = [(b'refs/heads/', b'H'),
                 (b'refs/remotes/origin/', b'RH'),
                 (b'refs/tags/', b'T')]
         _refs = {}
         for ref, sha in refs.items():
             for k, type_ in keys:
                 if ref.startswith(k):
                     _key = ref[len(k):]
                     if type_ == b'T':
                         obj = _repo.get_object(sha)
                         if isinstance(obj, Tag):
                             sha = _repo.get_object(sha).object[1]
                     _refs[_key] = [sha, type_]
                     break
         return _refs
 
     def _heads(self, reverse=False):
         refs = self._repo.get_refs()
         heads = {}
 
         for key, val in refs.items():
             for ref_key in [b'refs/heads/', b'refs/remotes/origin/']:
                 if key.startswith(ref_key):
                     n = key[len(ref_key):]
                     if n not in [b'HEAD']:
                         heads[n] = val
 
         return heads if reverse else dict((y, x) for x, y in heads.items())
 
     def get_changeset(self, revision=None):
         """
         Returns ``GitChangeset`` object representing commit from git repository
         at the given revision or head (most recent commit) if None given.
         """
-        if isinstance(revision, GitChangeset):
+        if isinstance(revision, changeset.GitChangeset):
             return revision
-        return GitChangeset(repository=self, revision=self._get_revision(revision))
+        return changeset.GitChangeset(repository=self, revision=self._get_revision(revision))
 
     def get_changesets(self, start=None, end=None, start_date=None,
            end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``GitChangeset`` objects from start to end (both
         are inclusive), in ascending date order (unless ``reverse`` is set).
 
         :param start: changeset ID, as str; first returned changeset
         :param end: changeset ID, as str; last returned changeset
         :param start_date: if specified, changesets with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, changesets with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, changesets not reachable from given
           branch would be filtered out from returned set
         :param reverse: if ``True``, returned generator would be reversed
           (meaning that returned changesets would have descending date order)
 
         :raise BranchDoesNotExistError: If given ``branch_name`` does not
             exist.
         :raise ChangesetDoesNotExistError: If changeset for given ``start`` or
           ``end`` could not be found.
 
         """
         if branch_name and branch_name not in self.branches:
             raise BranchDoesNotExistError("Branch '%s' not found"
                                           % branch_name)
         # actually we should check now if it's not an empty repo to not spaw
         # subprocess commands
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
 
         # %H at format means (full) commit hash, initial hashes are retrieved
         # in ascending date order
         cmd = ['log', '--date-order', '--reverse', '--pretty=format:%H']
         if max_revisions:
             cmd += ['--max-count=%s' % max_revisions]
         if start_date:
             cmd += ['--since', start_date.strftime('%m/%d/%y %H:%M:%S')]
         if end_date:
             cmd += ['--until', end_date.strftime('%m/%d/%y %H:%M:%S')]
         if branch_name:
             cmd.append(branch_name)
         else:
             cmd.append(settings.GIT_REV_FILTER)
 
         revs = self.run_git_command(cmd).splitlines()
         start_pos = 0
         end_pos = len(revs)
         if start:
             _start = self._get_revision(start)
             try:
                 start_pos = revs.index(_start)
             except ValueError:
                 pass
 
         if end is not None:
             _end = self._get_revision(end)
             try:
                 end_pos = revs.index(_end)
             except ValueError:
                 pass
 
         if None not in [start, end] and start_pos > end_pos:
             raise RepositoryError('start cannot be after end')
 
         if end_pos is not None:
             end_pos += 1
 
         revs = revs[start_pos:end_pos]
         if reverse:
             revs.reverse()
 
         return CollectionGenerator(self, revs)
 
     def get_diff_changesets(self, org_rev, other_repo, other_rev):
         """
         Returns lists of changesets that can be merged from this repo @org_rev
         to other_repo @other_rev
         ... and the other way
         ... and the ancestors that would be used for merge
 
         :param org_rev: the revision we want our compare to be made
         :param other_repo: repo object, most likely the fork of org_repo. It has
             all changesets that we need to obtain
         :param other_rev: revision we want out compare to be made on other_repo
         """
         org_changesets = []
         ancestors = None
         if org_rev == other_rev:
             other_changesets = []
         elif self != other_repo:
             gitrepo = Repo(self.path)
             SubprocessGitClient(thin_packs=False).fetch(other_repo.path, gitrepo)
 
             gitrepo_remote = Repo(other_repo.path)
             SubprocessGitClient(thin_packs=False).fetch(self.path, gitrepo_remote)
 
             revs = [
                 ascii_str(x.commit.id)
                 for x in gitrepo_remote.get_walker(include=[ascii_bytes(other_rev)],
                                                    exclude=[ascii_bytes(org_rev)])
             ]
             other_changesets = [other_repo.get_changeset(rev) for rev in reversed(revs)]
             if other_changesets:
                 ancestors = [other_changesets[0].parents[0].raw_id]
             else:
                 # no changesets from other repo, ancestor is the other_rev
                 ancestors = [other_rev]
 
             gitrepo.close()
             gitrepo_remote.close()
 
         else:
             so = self.run_git_command(
                 ['log', '--reverse', '--pretty=format:%H',
                  '-s', '%s..%s' % (org_rev, other_rev)]
             )
             other_changesets = [self.get_changeset(cs)
                           for cs in re.findall(r'[0-9a-fA-F]{40}', so)]
             so = self.run_git_command(
                 ['merge-base', org_rev, other_rev]
             )
             ancestors = [re.findall(r'[0-9a-fA-F]{40}', so)[0]]
 
         return other_changesets, org_changesets, ancestors
 
     def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
                  context=3):
         """
         Returns (git like) *diff*, as plain bytes text. Shows changes
         introduced by ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``. Due to limitations in Git, if
           value passed-in is greater than ``2**31-1``
           (``2147483647``), it will be set to ``2147483647``
           instead. If negative value is passed-in, it will be set to
           ``0`` instead.
         """
 
         # Git internally uses a signed long int for storing context
         # size (number of lines to show before and after the
         # differences). This can result in integer overflow, so we
         # ensure the requested context is smaller by one than the
         # number that would cause the overflow. It is highly unlikely
         # that a single file will contain that many lines, so this
         # kind of change should not cause any realistic consequences.
         overflowed_long_int = 2**31
 
         if context >= overflowed_long_int:
             context = overflowed_long_int - 1
 
         # Negative context values make no sense, and will result in
         # errors. Ensure this does not happen.
         if context < 0:
             context = 0
 
         flags = ['-U%s' % context, '--full-index', '--binary', '-p', '-M', '--abbrev=40']
         if ignore_whitespace:
             flags.append('-w')
 
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
 
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
 
         if rev1 == self.EMPTY_CHANGESET:
             rev2 = self.get_changeset(rev2).raw_id
             cmd = ['show'] + flags + [rev2]
         else:
             rev1 = self.get_changeset(rev1).raw_id
             rev2 = self.get_changeset(rev2).raw_id
             cmd = ['diff'] + flags + [rev1, rev2]
 
         if path:
             cmd += ['--', path]
 
         stdout, stderr = self._run_git_command(cmd, cwd=self.path)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if rev1 == self.EMPTY_CHANGESET:
             parts = stdout.split(b'\ndiff ', 1)
             if len(parts) > 1:
                 stdout = b'diff ' + parts[1]
         return stdout
 
     @LazyProperty
     def in_memory_changeset(self):
         """
         Returns ``GitInMemoryChangeset`` object for this repository.
         """
-        return GitInMemoryChangeset(self)
+        return inmemory.GitInMemoryChangeset(self)
 
     def clone(self, url, update_after_clone=True, bare=False):
         """
         Tries to clone changes from external location.
 
         :param update_after_clone: If set to ``False``, git won't checkout
           working directory
         :param bare: If set to ``True``, repository would be cloned into
           *bare* git repository (no working directory at all).
         """
         url = self._get_url(url)
         cmd = ['clone', '-q']
         if bare:
             cmd.append('--bare')
         elif not update_after_clone:
             cmd.append('--no-checkout')
         cmd += ['--', url, self.path]
         # If error occurs run_git_command raises RepositoryError already
         self.run_git_command(cmd)
 
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         url = self._get_url(url)
         cmd = ['pull', '--ff-only', url]
         # If error occurs run_git_command raises RepositoryError already
         self.run_git_command(cmd)
 
     def fetch(self, url):
         """
         Tries to pull changes from external location.
         """
         url = self._get_url(url)
         so = self.run_git_command(['ls-remote', '-h', url])
         cmd = ['fetch', url, '--']
         for line in (x for x in so.splitlines()):
             sha, ref = line.split('\t')
             cmd.append('+%s:%s' % (ref, ref))
         self.run_git_command(cmd)
 
     def _update_server_info(self):
         """
         runs gits update-server-info command in this repo instance
         """
         try:
             update_server_info(self._repo)
         except OSError as e:
             if e.errno not in [errno.ENOENT, errno.EROFS]:
                 raise
             # Workaround for dulwich crashing on for example its own dulwich/tests/data/repos/simple_merge.git/info/refs.lock
             log.error('Ignoring %s running update-server-info: %s', type(e).__name__, e)
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
-        return GitWorkdir(self)
+        return workdir.GitWorkdir(self)
 
     def get_config_value(self, section, name, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
 
         def gen_configs():
             for path in config_file + self._config_files:
                 try:
                     yield ConfigFile.from_path(path)
                 except (IOError, OSError, ValueError):
                     continue
 
         for config in gen_configs():
             try:
                 value = config.get(section, name)
             except KeyError:
                 continue
             return None if value is None else safe_str(value)
         return None
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'name', config_file)
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'email', config_file)

kallithea/lib/vcs/backends/hg/inmemory.py

@@ -1,110 +1,111 b''
 import datetime
 
 import mercurial.context
 import mercurial.node
 
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import ascii_str, safe_bytes, safe_str
 
+from . import repository
+
 
 class MercurialInMemoryChangeset(BaseInMemoryChangeset):
 
     def commit(self, message, author, parents=None, branch=None, date=None,
                **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         self.check_integrity(parents)
 
         if not isinstance(message, str):
             raise RepositoryError('message must be a str - got %r' % type(message))
         if not isinstance(author, str):
             raise RepositoryError('author must be a str - got %r' % type(author))
 
-        from .repository import MercurialRepository
         if branch is None:
-            branch = MercurialRepository.DEFAULT_BRANCH_NAME
+            branch = repository.MercurialRepository.DEFAULT_BRANCH_NAME
         kwargs[b'branch'] = safe_bytes(branch)
 
         def filectxfn(_repo, memctx, bytes_path):
             """
             Callback from Mercurial, returning ctx to commit for the given
             path.
             """
             path = safe_str(bytes_path)
 
             # check if this path is removed
             if path in (node.path for node in self.removed):
                 return None
 
             # check if this path is added
             for node in self.added:
                 if node.path == path:
                     return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
                         data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
 
             # or changed
             for node in self.changed:
                 if node.path == path:
                     return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
                         data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
 
             raise RepositoryError("Given path haven't been marked as added, "
                                   "changed or removed (%s)" % path)
 
         parents = [None, None]
         for i, parent in enumerate(self.parents):
             if parent is not None:
                 parents[i] = parent._ctx.node()
 
         if date and isinstance(date, datetime.datetime):
             date = safe_bytes(date.strftime('%a, %d %b %Y %H:%M:%S'))
 
         commit_ctx = mercurial.context.memctx(
             repo=self.repository._repo,
             parents=parents,
             text=b'',
             files=[safe_bytes(x) for x in self.get_paths()],
             filectxfn=filectxfn,
             user=safe_bytes(author),
             date=date,
             extra=kwargs)
 
         # injecting given _repo params
         commit_ctx._text = safe_bytes(message)
         commit_ctx._user = safe_bytes(author)
         commit_ctx._date = date
 
         # TODO: Catch exceptions!
         n = self.repository._repo.commitctx(commit_ctx)
         # Returns mercurial node
         self._commit_ctx = commit_ctx  # For reference
         # Update vcs repository object & recreate mercurial _repo
         # new_ctx = self.repository._repo[node]
         # new_tip = ascii_str(self.repository.get_changeset(new_ctx.hex()))
         self.repository.revisions.append(ascii_str(mercurial.node.hex(n)))
         self._repo = self.repository._get_repo(create=False)
         self.repository.branches = self.repository._get_branches()
         tip = self.repository.get_changeset()
         self.reset()
         return tip

kallithea/lib/vcs/backends/hg/repository.py

@@ -1,665 +1,663 b''
 # -*- coding: utf-8 -*-
 """
     vcs.backends.hg.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Mercurial repository implementation.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import datetime
 import logging
 import os
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 
 import mercurial.commands
 import mercurial.error
 import mercurial.exchange
 import mercurial.hg
 import mercurial.hgweb
 import mercurial.httppeer
 import mercurial.localrepo
 import mercurial.match
 import mercurial.mdiff
 import mercurial.node
 import mercurial.patch
 import mercurial.scmutil
 import mercurial.sshpeer
 import mercurial.tags
 import mercurial.ui
 import mercurial.unionrepo
 import mercurial.util
 
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.exceptions import (BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
                                           TagDoesNotExistError, VCSError)
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_str
 from kallithea.lib.vcs.utils.helpers import get_urllib_request_handlers
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath
 
-from .changeset import MercurialChangeset
-from .inmemory import MercurialInMemoryChangeset
-from .workdir import MercurialWorkdir
+from . import changeset, inmemory, workdir
 
 
 log = logging.getLogger(__name__)
 
 
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
     scm = 'hg'
 
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
                  update_after_clone=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
 
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param baseui=None: user data
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
         """
 
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
         self.path = abspath(repo_path)
         self.baseui = baseui or mercurial.ui.ui()
         # We've set path and ui, now we can set _repo itself
         self._repo = self._get_repo(create, src_url, update_after_clone)
 
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         # TODO: Following raises errors when using InMemoryChangeset...
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
 
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
 
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
 
     @LazyProperty
     def branches(self):
         return self._get_branches()
 
     @LazyProperty
     def closed_branches(self):
         return self._get_branches(normal=False, closed=True)
 
     @LazyProperty
     def allbranches(self):
         """
         List all branches, including closed branches.
         """
         return self._get_branches(closed=True)
 
     def _get_branches(self, normal=True, closed=False):
         """
         Gets branches for this repository
         Returns only not closed branches by default
 
         :param closed: return also closed branches for mercurial
         :param normal: return also normal branches
         """
 
         if self._empty:
             return {}
 
         bt = OrderedDict()
         for bn, _heads, node, isclosed in sorted(self._repo.branchmap().iterbranches()):
             if isclosed:
                 if closed:
                     bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
             else:
                 if normal:
                     bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
         return bt
 
     @LazyProperty
     def tags(self):
         """
         Gets tags for this repository
         """
         return self._get_tags()
 
     def _get_tags(self):
         if self._empty:
             return {}
 
         return OrderedDict(sorted(
             ((safe_str(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo.tags().items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
 
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         local = kwargs.setdefault('local', False)
 
         if message is None:
             message = "Added tag %s for changeset %s" % (name,
                 changeset.short_id)
 
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
 
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), changeset._ctx.node(), safe_bytes(message), local, safe_bytes(user), date)
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
 
         # Reinitialize tags
         self.tags = self._get_tags()
         tag_id = self.tags[name]
 
         return self.get_changeset(revision=tag_id)
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         if message is None:
             message = "Removed tag %s" % name
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
         local = False
 
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), mercurial.commands.nullid, safe_bytes(message), local, safe_bytes(user), date)
             self.tags = self._get_tags()
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
 
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return self._get_bookmarks()
 
     def _get_bookmarks(self):
         if self._empty:
             return {}
 
         return OrderedDict(sorted(
             ((safe_str(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo._bookmarks.items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
 
     def _get_all_revisions(self):
         return [ascii_str(self._repo[x].hex()) for x in self._repo.filtered(b'visible').changelog.revs()]
 
     def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
                   context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``. If negative value is passed-in, it will be
           set to ``0`` instead.
         """
 
         # Negative context values make no sense, and will result in
         # errors. Ensure this does not happen.
         if context < 0:
             context = 0
 
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
 
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
 
         # Check if given revisions are present at repository (may raise
         # ChangesetDoesNotExistError)
         if rev1 != self.EMPTY_CHANGESET:
             self.get_changeset(rev1)
         self.get_changeset(rev2)
         if path:
             file_filter = mercurial.match.exact([safe_bytes(path)])
         else:
             file_filter = None
 
         return b''.join(mercurial.patch.diff(self._repo, rev1, rev2, match=file_filter,
                           opts=mercurial.mdiff.diffopts(git=True,
                                         showfunc=True,
                                         ignorews=ignore_whitespace,
                                         context=context)))
 
     @classmethod
     def _check_url(cls, url, repoui=None):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         url = safe_bytes(url)
         if os.path.isdir(url) or url.startswith(b'file:'):
             return True
 
         if url.startswith(b'ssh:'):
             # in case of invalid uri or authentication issues, sshpeer will
             # throw an exception.
             mercurial.sshpeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             return True
 
         url_prefix = None
         if b'+' in url[:url.find(b'://')]:
             url_prefix, url = url.split(b'+', 1)
 
         url_obj = mercurial.util.url(url)
         test_uri, handlers = get_urllib_request_handlers(url_obj)
 
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
 
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         req = urllib.request.Request(
             "%s?%s" % (
                 safe_str(test_uri),
                 urllib.parse.urlencode({
                     'cmd': 'between',
                     'pairs': "%s-%s" % ('0' * 40, '0' * 40),
                 })
             ))
 
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         if not url_prefix: # skip git+http://... etc
             # now check if it's a proper hg repo
             try:
                 mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             except Exception as e:
                 raise urllib.error.URLError(
                     "url [%s] does not look like an hg repo org_exc: %s"
                     % (cleaned_uri, e))
 
         return True
 
     def _get_repo(self, create, src_url=None, update_after_clone=False):
         """
         Function will check for mercurial repository in given path and return
         a localrepo object. If there is no repository in that path it will
         raise an exception unless ``create`` parameter is set to True - in
         that case repository would be created and returned.
         If ``src_url`` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to ``update_after_clone`` flag
         """
         try:
             if src_url:
                 url = safe_bytes(self._get_url(src_url))
                 opts = {}
                 if not update_after_clone:
                     opts.update({'noupdate': True})
                 MercurialRepository._check_url(url, self.baseui)
                 mercurial.commands.clone(self.baseui, url, safe_bytes(self.path), **opts)
 
                 # Don't try to create if we've already cloned repo
                 create = False
             return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)
         except (mercurial.error.Abort, mercurial.error.RepoError) as err:
             if create:
                 msg = "Cannot create repository at %s. Original error was %s" \
                     % (self.name, err)
             else:
                 msg = "Not valid repository at %s. Original error was %s" \
                     % (self.name, err)
             raise RepositoryError(msg)
 
     @LazyProperty
     def in_memory_changeset(self):
-        return MercurialInMemoryChangeset(self)
+        return inmemory.MercurialInMemoryChangeset(self)
 
     @LazyProperty
     def description(self):
         _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)
         return safe_str(_desc or b'unknown')
 
     @LazyProperty
     def contact(self):
         return safe_str(mercurial.hgweb.common.get_contact(self._repo.ui.config)
                             or b'Unknown')
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
 
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             # fallback to filesystem
             cl_path = os.path.join(self.path, '.hg', "00changelog.i")
             st_path = os.path.join(self.path, '.hg', "store")
             if os.path.exists(cl_path):
                 return os.stat(cl_path).st_mtime
             else:
                 return os.stat(st_path).st_mtime
 
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
 
         :param revision: str or int or None
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
 
         if revision in [-1, None]:
             revision = b'tip'
         elif isinstance(revision, str):
             revision = safe_bytes(revision)
 
         try:
             if isinstance(revision, int):
                 return ascii_str(self._repo[revision].hex())
             return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
         except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
             msg = "Revision %r does not exist for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
         except (LookupError, ):
             msg = "Ambiguous identifier `%s` for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
 
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns revision number for the given reference.
         """
         if ref_type == 'rev' and not ref_name.strip('0'):
             return self.EMPTY_CHANGESET
         # lookup up the exact node id
         _revset_predicates = {
                 'branch': 'branch',
                 'book': 'bookmark',
                 'tag': 'tag',
                 'rev': 'id',
             }
         # avoid expensive branch(x) iteration over whole repo
         rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]
         try:
             revs = self._repo.revs(rev_spec, ref_name, ref_name)
         except LookupError:
             msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         except mercurial.error.RepoLookupError:
             msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         if revs:
             revision = revs.last()
         else:
             # TODO: just report 'not found'?
             revision = ref_name
 
         return self._get_revision(revision)
 
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist(b"web", b"allow_archive",
                                          untrusted=True)
         for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
                 yield {"type": safe_str(name), "extension": ext, "node": archive_name}
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, fall back to
         filesystem (``file:///``) schema.
         """
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.request.pathname2url(url)
         return url
 
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
-        return MercurialChangeset(repository=self, revision=self._get_revision(revision))
+        return changeset.MercurialChangeset(repository=self, revision=self._get_revision(revision))
 
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
 
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
         start_raw_id = self._get_revision(start)
         start_pos = None if start is None else self.revisions.index(start_raw_id)
         end_raw_id = self._get_revision(end)
         end_pos = None if end is None else self.revisions.index(end_raw_id)
 
         if start_pos is not None and end_pos is not None and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
 
         if branch_name and branch_name not in self.allbranches:
             msg = "Branch %r not found in %s" % (branch_name, self.name)
             raise BranchDoesNotExistError(msg)
         if end_pos is not None:
             end_pos += 1
         # filter branches
         filter_ = []
         if branch_name:
             filter_.append(b'branch("%s")' % safe_bytes(branch_name))
         if start_date:
             filter_.append(b'date(">%s")' % safe_bytes(str(start_date)))
         if end_date:
             filter_.append(b'date("<%s")' % safe_bytes(str(end_date)))
         if filter_ or max_revisions:
             if filter_:
                 revspec = b' and '.join(filter_)
             else:
                 revspec = b'all()'
             if max_revisions:
                 revspec = b'limit(%s, %d)' % (revspec, max_revisions)
             revisions = mercurial.scmutil.revrange(self._repo, [revspec])
         else:
             revisions = self.revisions
 
         # this is very much a hack to turn this into a list; a better solution
         # would be to get rid of this function entirely and use revsets
         revs = list(revisions)[start_pos:end_pos]
         if reverse:
             revs.reverse()
 
         return CollectionGenerator(self, revs)
 
     def get_diff_changesets(self, org_rev, other_repo, other_rev):
         """
         Returns lists of changesets that can be merged from this repo @org_rev
         to other_repo @other_rev
         ... and the other way
         ... and the ancestors that would be used for merge
 
         :param org_rev: the revision we want our compare to be made
         :param other_repo: repo object, most likely the fork of org_repo. It has
             all changesets that we need to obtain
         :param other_rev: revision we want out compare to be made on other_repo
         """
         ancestors = None
         if org_rev == other_rev:
             org_changesets = []
             other_changesets = []
 
         else:
             # case two independent repos
             if self != other_repo:
                 hgrepo = mercurial.unionrepo.makeunionrepository(other_repo.baseui,
                                                        safe_bytes(other_repo.path),
                                                        safe_bytes(self.path))
                 # all ancestors of other_rev will be in other_repo and
                 # rev numbers from hgrepo can be used in other_repo - org_rev ancestors cannot
 
             # no remote compare do it on the same repository
             else:
                 hgrepo = other_repo._repo
 
             ancestors = [ascii_str(hgrepo[ancestor].hex()) for ancestor in
                          hgrepo.revs(b"id(%s) & ::id(%s)", ascii_bytes(other_rev), ascii_bytes(org_rev))]
             if ancestors:
                 log.debug("shortcut found: %s is already an ancestor of %s", other_rev, org_rev)
             else:
                 log.debug("no shortcut found: %s is not an ancestor of %s", other_rev, org_rev)
                 ancestors = [ascii_str(hgrepo[ancestor].hex()) for ancestor in
                              hgrepo.revs(b"heads(::id(%s) & ::id(%s))", ascii_bytes(org_rev), ascii_bytes(other_rev))] # FIXME: expensive!
 
             other_changesets = [
                 other_repo.get_changeset(rev)
                 for rev in hgrepo.revs(
                     b"ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                     ascii_bytes(other_rev), ascii_bytes(org_rev), ascii_bytes(org_rev))
             ]
             org_changesets = [
                 self.get_changeset(ascii_str(hgrepo[rev].hex()))
                 for rev in hgrepo.revs(
                     b"ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                     ascii_bytes(org_rev), ascii_bytes(other_rev), ascii_bytes(other_rev))
             ]
 
         return other_changesets, org_changesets, ancestors
 
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
         try:
             mercurial.exchange.pull(self._repo, other, heads=None, force=None)
         except mercurial.error.Abort as err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
-        return MercurialWorkdir(self)
+        return workdir.MercurialWorkdir(self)
 
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
 
         config = self._repo.ui
         if config_file:
             config = mercurial.ui.ui()
             for path in config_file:
                 config.readconfig(safe_bytes(path))
         value = config.config(safe_bytes(section), safe_bytes(name))
         return value if value is None else safe_str(value)
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_name(username)
         return None
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_email(username)
         return None

kallithea/lib/vcs/utils/helpers.py

@@ -1,243 +1,243 b''
 """
 Utilities aimed to help achieve mostly basic tasks.
 """
 
 import datetime
 import logging
 import os
 import re
 import time
 import urllib.request
 
 import mercurial.url
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import ClassNotFound, guess_lexer_for_filename
 
+from kallithea.lib.vcs import backends
 from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.lib.vcs.utils.paths import abspath
 
 
 ALIASES = ['hg', 'git']
 
 
 def get_scm(path, search_up=False, explicit_alias=None):
     """
     Returns one of alias from ``ALIASES`` (in order of precedence same as
     shortcuts given in ``ALIASES``) and top working dir path for the given
     argument. If no scm-specific directory is found or more than one scm is
     found at that directory, ``VCSError`` is raised.
 
     :param search_up: if set to ``True``, this function would try to
       move up to parent directory every time no scm is recognized for the
       currently checked path. Default: ``False``.
     :param explicit_alias: can be one of available backend aliases, when given
       it will return given explicit alias in repositories under more than one
       version control, if explicit_alias is different than found it will raise
       VCSError
     """
     if not os.path.isdir(path):
         raise VCSError("Given path %s is not a directory" % path)
 
     while True:
         found_scms = [(scm, path) for scm in get_scms_for_path(path)]
         if found_scms or not search_up:
             break
         newpath = abspath(path, '..')
         if newpath == path:
             break
         path = newpath
 
     if len(found_scms) > 1:
         for scm in found_scms:
             if scm[0] == explicit_alias:
                 return scm
         raise VCSError('More than one [%s] scm found at given path %s'
                        % (', '.join((x[0] for x in found_scms)), path))
 
     if len(found_scms) == 0:
         raise VCSError('No scm found at given path %s' % path)
 
     return found_scms[0]
 
 
 def get_scms_for_path(path):
     """
     Returns all scm's found at the given path. If no scm is recognized
     - empty list is returned.
 
     :param path: path to directory which should be checked. May be callable.
 
     :raises VCSError: if given ``path`` is not a directory
     """
-    from kallithea.lib.vcs.backends import get_backend
     if hasattr(path, '__call__'):
         path = path()
     if not os.path.isdir(path):
         raise VCSError("Given path %r is not a directory" % path)
 
     result = []
     for key in ALIASES:
         # find .hg / .git
         dirname = os.path.join(path, '.' + key)
         if os.path.isdir(dirname):
             result.append(key)
             continue
         # find rm__.hg / rm__.git too - left overs from old method for deleting
         dirname = os.path.join(path, 'rm__.' + key)
         if os.path.isdir(dirname):
             return result
         # We still need to check if it's not bare repository as
         # bare repos don't have working directories
         try:
-            get_backend(key)(path)
+            backends.get_backend(key)(path)
             result.append(key)
             continue
         except RepositoryError:
             # Wrong backend
             pass
         except VCSError:
             # No backend at all
             pass
     return result
 
 
 def get_highlighted_code(name, code, type='terminal'):
     """
     If pygments are available on the system
     then returned output is colored. Otherwise
     unchanged content is returned.
     """
     try:
         lexer = guess_lexer_for_filename(name, code)
         formatter = TerminalFormatter()
         content = highlight(code, lexer, formatter)
     except ClassNotFound:
         logging.debug("Couldn't guess Lexer, will not use pygments.")
         content = code
     return content
 
 
 def parse_changesets(text):
     """
     Returns dictionary with *start*, *main* and *end* ids.
 
     Examples::
 
         >>> parse_changesets('aaabbb')
         {'start': None, 'main': 'aaabbb', 'end': None}
         >>> parse_changesets('aaabbb..cccddd')
         {'start': 'aaabbb', 'end': 'cccddd', 'main': None}
 
     """
     text = text.strip()
     CID_RE = r'[a-zA-Z0-9]+'
     if '..' not in text:
         m = re.match(r'^(?P<cid>%s)$' % CID_RE, text)
         if m:
             return {
                 'start': None,
                 'main': text,
                 'end': None,
             }
     else:
         RE = r'^(?P<start>%s)?\.{2,3}(?P<end>%s)?$' % (CID_RE, CID_RE)
         m = re.match(RE, text)
         if m:
             result = m.groupdict()
             result['main'] = None
             return result
     raise ValueError("IDs not recognized")
 
 
 def parse_datetime(text):
     """
     Parses given text and returns ``datetime.datetime`` instance or raises
     ``ValueError``.
 
     :param text: string of desired date/datetime or something more verbose,
       like *yesterday*, *2weeks 3days*, etc.
     """
 
     text = text.strip().lower()
 
     INPUT_FORMATS = (
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M',
         '%Y-%m-%d',
         '%m/%d/%Y %H:%M:%S',
         '%m/%d/%Y %H:%M',
         '%m/%d/%Y',
         '%m/%d/%y %H:%M:%S',
         '%m/%d/%y %H:%M',
         '%m/%d/%y',
     )
     for format in INPUT_FORMATS:
         try:
             return datetime.datetime(*time.strptime(text, format)[:6])
         except ValueError:
             pass
 
     # Try descriptive texts
     if text == 'tomorrow':
         future = datetime.datetime.now() + datetime.timedelta(days=1)
         args = future.timetuple()[:3] + (23, 59, 59)
         return datetime.datetime(*args)
     elif text == 'today':
         return datetime.datetime(*datetime.datetime.today().timetuple()[:3])
     elif text == 'now':
         return datetime.datetime.now()
     elif text == 'yesterday':
         past = datetime.datetime.now() - datetime.timedelta(days=1)
         return datetime.datetime(*past.timetuple()[:3])
     else:
         days = 0
         matched = re.match(
             r'^((?P<weeks>\d+) ?w(eeks?)?)? ?((?P<days>\d+) ?d(ays?)?)?$', text)
         if matched:
             groupdict = matched.groupdict()
             if groupdict['days']:
                 days += int(matched.groupdict()['days'])
             if groupdict['weeks']:
                 days += int(matched.groupdict()['weeks']) * 7
             past = datetime.datetime.now() - datetime.timedelta(days=days)
             return datetime.datetime(*past.timetuple()[:3])
 
     raise ValueError('Wrong date: "%s"' % text)
 
 
 def get_dict_for_attrs(obj, attrs):
     """
     Returns dictionary for each attribute from given ``obj``.
     """
     data = {}
     for attr in attrs:
         data[attr] = getattr(obj, attr)
     return data
 
 def get_urllib_request_handlers(url_obj):
     handlers = []
     test_uri, authinfo = url_obj.authinfo()
 
     if authinfo:
         # authinfo is a tuple (realm, uris, user, password) where 'uris' itself
         # is a tuple of URIs.
         # If url_obj is obtained via mercurial.util.url, the obtained authinfo
         # values will be bytes, e.g.
         #    (None, (b'http://127.0.0.1/repo', b'127.0.0.1'), b'user', b'pass')
         # However, urllib expects strings, not bytes, so we must convert them.
 
         # create a password manager
         passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
         passmgr.add_password(
             safe_str(authinfo[0]) if authinfo[0] else None, # realm
             tuple(safe_str(x) for x in authinfo[1]),        # uris
             safe_str(authinfo[2]),                          # user
             safe_str(authinfo[3]),                          # password
         )
 
         handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                          mercurial.url.httpdigestauthhandler(passmgr)))
 
     return test_uri, handlers

kallithea/model/db.py

@@ -1,2303 +1,2302 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.db
 ~~~~~~~~~~~~~~~~~~
 
 Database Models for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 08, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import base64
 import collections
 import datetime
 import functools
 import hashlib
 import logging
 import os
 import time
 import traceback
 
 import ipaddr
 import sqlalchemy
+import urlobject
 from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, String, Unicode, UnicodeText, UniqueConstraint
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
 from tg.i18n import lazy_ugettext as _
 from webob.exc import HTTPNotFound
 
 import kallithea
 from kallithea.lib import ext_json, ssh, webutils
 from kallithea.lib.exceptions import DefaultUserException
 from kallithea.lib.utils2 import asbool, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, urlreadable
 from kallithea.lib.vcs import get_backend, get_repo
-from kallithea.lib.vcs.backends.base import EmptyChangeset
+from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.utils import author_email, author_name
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.model import meta
 
 
 log = logging.getLogger(__name__)
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 class BaseDbModel(object):
     """
     Base Model for all classes
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         # Note: not a normal dict - iterator gives "users.firstname", but keys gives "firstname"
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
 
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.items():
                 d[k] = val
         return d
 
     def get_appstruct(self):
         """return list with keys and values tuples corresponding
         to this model data """
 
         return [
             (k, getattr(self, k))
             for k in self._get_keys()
         ]
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return meta.Session().query(cls)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def guess_instance(cls, value, callback=None):
         """Haphazardly attempt to convert `value` to a `cls` instance.
 
         If `value` is None or already a `cls` instance, return it. If `value`
         is a number (or looks like one if you squint just right), assume it's
         a database primary key and let SQLAlchemy sort things out. Otherwise,
         fall back to resolving it using `callback` (if specified); this could
         e.g. be a function that looks up instances by name (though that won't
         work if the name begins with a digit). Otherwise, raise Exception.
         """
 
         if value is None:
             return None
         if isinstance(value, cls):
             return value
         if isinstance(value, int):
             return cls.get(value)
         if isinstance(value, str) and value.isdigit():
             return cls.get(int(value))
         if callback is not None:
             return callback(value)
 
         raise Exception(
             'given object must be int, long or Instance of %s '
             'got %s, no callback provided' % (cls, type(value))
         )
 
     @classmethod
     def get_or_404(cls, id_):
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound
 
         res = cls.query().get(id_)
         if res is None:
             raise HTTPNotFound
         return res
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         meta.Session().delete(obj)
 
     def __repr__(self):
         return '<DB:%s>' % (self.__class__.__name__)
 
 
 _table_args_default_dict = {'extend_existing': True,
                             'mysql_engine': 'InnoDB',
                             'sqlite_autoincrement': True,
                            }
 
 class Setting(meta.Base, BaseDbModel):
     __tablename__ = 'settings'
     __table_args__ = (
         _table_args_default_dict,
     )
 
     SETTINGS_TYPES = {
         'str': safe_bytes,
         'int': safe_int,
         'unicode': safe_str,
         'bool': asbool,
         'list': functools.partial(aslist, sep=',')
     }
 
     app_settings_id = Column(Integer(), primary_key=True)
     app_settings_name = Column(String(255), nullable=False, unique=True)
     _app_settings_value = Column("app_settings_value", Unicode(4096), nullable=False)
     _app_settings_type = Column("app_settings_type", String(255), nullable=True) # FIXME: not nullable?
 
     def __init__(self, key='', val='', type='unicode'):
         self.app_settings_name = key
         self.app_settings_value = val
         self.app_settings_type = type
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert isinstance(val, str)
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         _type = self.app_settings_type
         converter = self.SETTINGS_TYPES.get(_type) or self.SETTINGS_TYPES['unicode']
         return converter(v)
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use str in app_settings_value
         """
         self._app_settings_value = safe_str(val)
 
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
 
     @app_settings_type.setter
     def app_settings_type(self, val):
         if val not in self.SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (list(self.SETTINGS_TYPES), val))
         self._app_settings_type = val
 
     def __repr__(self):
         return "<%s %s.%s=%r>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_type, self.app_settings_value
         )
 
     @classmethod
     def get_by_name(cls, key):
         return cls.query() \
             .filter(cls.app_settings_name == key).scalar()
 
     @classmethod
     def get_by_name_or_create(cls, key, val='', type='unicode'):
         res = cls.get_by_name(key)
         if res is None:
             res = cls(key, val, type)
         return res
 
     @classmethod
     def create_or_update(cls, key, val=None, type=None):
         """
         Creates or updates Kallithea setting. If updates are triggered, it will only
         update parameters that are explicitly set. 'None' values will be skipped.
 
         :param key:
         :param val:
         :param type:
         :return:
         """
         res = cls.get_by_name(key)
         if res is None:
             # new setting
             val = val if val is not None else ''
             type = type if type is not None else 'unicode'
             res = cls(key, val, type)
             meta.Session().add(res)
         else:
             if val is not None:
                 # update if set
                 res.app_settings_value = val
             if type is not None:
                 # update if set
                 res.app_settings_type = type
         return res
 
     @classmethod
     def get_app_settings(cls):
 
         ret = cls.query()
         if ret is None:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings[each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_auth_settings(cls):
         ret = cls.query() \
                 .filter(cls.app_settings_name.startswith('auth_')).all()
         fd = {}
         for row in ret:
             fd[row.app_settings_name] = row.app_settings_value
         return fd
 
     @classmethod
     def get_default_repo_settings(cls, strip_prefix=False):
         ret = cls.query() \
                 .filter(cls.app_settings_name.startswith('default_')).all()
         fd = {}
         for row in ret:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             fd.update({key: row.app_settings_value})
 
         return fd
 
     @classmethod
     def get_server_info(cls):
         import platform
 
         import pkg_resources
 
         from kallithea.lib.utils import check_git_version
         mods = [(p.project_name, p.version) for p in pkg_resources.working_set]
         info = {
             'modules': sorted(mods, key=lambda k: k[0].lower()),
             'py_version': platform.python_version(),
             'platform': platform.platform(),
             'kallithea_version': kallithea.__version__,
             'git_version': str(check_git_version()),
             'git_path': kallithea.CONFIG.get('git_path')
         }
         return info
 
 
 class Ui(meta.Base, BaseDbModel):
     __tablename__ = 'ui'
     __table_args__ = (
         Index('ui_ui_section_ui_key_idx', 'ui_section', 'ui_key'),
         UniqueConstraint('ui_section', 'ui_key'),
         _table_args_default_dict,
     )
 
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
 
     ui_id = Column(Integer(), primary_key=True)
     ui_section = Column(String(255), nullable=False)
     ui_key = Column(String(255), nullable=False)
     ui_value = Column(String(255), nullable=True) # FIXME: not nullable?
     ui_active = Column(Boolean(), nullable=False, default=True)
 
     @classmethod
     def get_by_key(cls, section, key):
         """ Return specified Ui object, or None if not found. """
         return cls.query().filter_by(ui_section=section, ui_key=key).scalar()
 
     @classmethod
     def get_or_create(cls, section, key):
         """ Return specified Ui object, creating it if necessary. """
         setting = cls.get_by_key(section, key)
         if setting is None:
             setting = cls(ui_section=section, ui_key=key)
             meta.Session().add(setting)
         return setting
 
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
         q = q.filter(cls.ui_section == 'hooks')
         q = q.order_by(cls.ui_section, cls.ui_key)
         return q.all()
 
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
         q = q.filter(cls.ui_section == 'hooks')
         q = q.order_by(cls.ui_section, cls.ui_key)
         return q.all()
 
     @classmethod
     def get_repos_location(cls):
         return cls.get_by_key('paths', '/').ui_value
 
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_or_create('hooks', key)
         new_ui.ui_active = True
         new_ui.ui_value = val
 
     def __repr__(self):
         return '<%s %s.%s=%r>' % (
             self.__class__.__name__,
             self.ui_section, self.ui_key, self.ui_value)
 
 
 class User(meta.Base, BaseDbModel):
     __tablename__ = 'users'
     __table_args__ = (
         Index('u_username_idx', 'username'),
         Index('u_email_idx', 'email'),
         _table_args_default_dict,
     )
 
     DEFAULT_USER_NAME = 'default'
     DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
     # The name of the default auth type in extern_type, 'internal' lives in auth_internal.py
     DEFAULT_AUTH_TYPE = 'internal'
 
     user_id = Column(Integer(), primary_key=True)
     username = Column(String(255), nullable=False, unique=True)
     password = Column(String(255), nullable=False)
     active = Column(Boolean(), nullable=False, default=True)
     admin = Column(Boolean(), nullable=False, default=False)
     name = Column("firstname", Unicode(255), nullable=False)
     lastname = Column(Unicode(255), nullable=False)
     _email = Column("email", String(255), nullable=True, unique=True) # FIXME: not nullable?
     last_login = Column(DateTime(timezone=False), nullable=True)
     extern_type = Column(String(255), nullable=True) # FIXME: not nullable?
     extern_name = Column(String(255), nullable=True) # FIXME: not nullable?
     api_key = Column(String(255), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _user_data = Column("user_data", LargeBinary(), nullable=True)  # JSON data # FIXME: not nullable?
 
     user_log = relationship('UserLog')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relationship('Repository')
     repo_groups = relationship('RepoGroup')
     user_groups = relationship('UserGroup')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
 
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
     repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all')
 
     group_member = relationship('UserGroupMember', cascade='all')
 
     # comments created by this user
     user_comments = relationship('ChangesetComment', cascade='all')
     # extra emails for this user
     user_emails = relationship('UserEmailMap', cascade='all')
     # extra API keys
     user_api_keys = relationship('UserApiKeys', cascade='all')
     ssh_keys = relationship('UserSshKeys', cascade='all')
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
     @property
     def firstname(self):
         # alias for future
         return self.name
 
     @property
     def emails(self):
         other = UserEmailMap.query().filter(UserEmailMap.user == self).all()
         return [self.email] + [x.email for x in other]
 
     @property
     def api_keys(self):
         other = UserApiKeys.query().filter(UserApiKeys.user == self).all()
         return [self.api_key] + [x.api_key for x in other]
 
     @property
     def ip_addresses(self):
         ret = UserIpMap.query().filter(UserIpMap.user == self).all()
         return [x.ip_addr for x in ret]
 
     @property
     def full_name(self):
         return '%s %s' % (self.firstname, self.lastname)
 
     @property
     def full_name_or_username(self):
         """
         Show full name.
         If full name is not set, fall back to username.
         """
         return ('%s %s' % (self.firstname, self.lastname)
                 if (self.firstname and self.lastname) else self.username)
 
     @property
     def full_name_and_username(self):
         """
         Show full name and username as 'Firstname Lastname (username)'.
         If full name is not set, fall back to username.
         """
         return ('%s %s (%s)' % (self.firstname, self.lastname, self.username)
                 if (self.firstname and self.lastname) else self.username)
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.firstname, self.lastname, self.email)
 
     @property
     def short_contact(self):
         return '%s %s' % (self.firstname, self.lastname)
 
     @property
     def is_admin(self):
         return self.admin
 
     @hybrid_property
     def is_default_user(self):
         return self.username == User.DEFAULT_USER_NAME
 
     @hybrid_property
     def user_data(self):
         if not self._user_data:
             return {}
 
         try:
             return ext_json.loads(self._user_data)
         except TypeError:
             return {}
 
     @user_data.setter
     def user_data(self, val):
         try:
             self._user_data = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
 
     def __repr__(self):
         return "<%s %s: %r>" % (self.__class__.__name__, self.user_id, self.username)
 
     @classmethod
     def guess_instance(cls, value):
         return super(User, cls).guess_instance(value, User.get_by_username)
 
     @classmethod
     def get_or_404(cls, id_, allow_default=True):
         '''
         Overridden version of BaseDbModel.get_or_404, with an extra check on
         the default user.
         '''
         user = super(User, cls).get_or_404(id_)
         if not allow_default and user.is_default_user:
             raise DefaultUserException()
         return user
 
     @classmethod
     def get_by_username_or_email(cls, username_or_email, case_insensitive=True):
         """
         For anything that looks like an email address, look up by the email address (matching
         case insensitively).
         For anything else, try to look up by the user name.
 
         This assumes no normal username can have '@' symbol.
         """
         if '@' in username_or_email:
             return User.get_by_email(username_or_email)
         else:
             return User.get_by_username(username_or_email, case_insensitive=case_insensitive)
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(sqlalchemy.func.lower(cls.username) == sqlalchemy.func.lower(username))
         else:
             q = cls.query().filter(cls.username == username)
         return q.scalar()
 
     @classmethod
     def get_by_api_key(cls, api_key, fallback=True):
         if len(api_key) != 40 or not api_key.isalnum():
             return None
 
         q = cls.query().filter(cls.api_key == api_key)
         res = q.scalar()
 
         if fallback and not res:
             # fallback to additional keys
             _res = UserApiKeys.query().filter_by(api_key=api_key, is_expired=False).first()
             if _res:
                 res = _res.user
         if res is None or not res.active or res.is_default_user:
             return None
         return res
 
     @classmethod
     def get_by_email(cls, email, cache=False):
         q = cls.query().filter(sqlalchemy.func.lower(cls.email) == sqlalchemy.func.lower(email))
         ret = q.scalar()
         if ret is None:
             q = UserEmailMap.query()
             # try fetching in alternate email map
             q = q.filter(sqlalchemy.func.lower(UserEmailMap.email) == sqlalchemy.func.lower(email))
             q = q.options(joinedload(UserEmailMap.user))
             ret = getattr(q.scalar(), 'user', None)
 
         return ret
 
     @classmethod
     def get_from_cs_author(cls, author):
         """
         Tries to get User objects out of commit author string
         """
         # Valid email in the attribute passed, see if they're in the system
         _email = author_email(author)
         if _email:
             user = cls.get_by_email(_email)
             if user is not None:
                 return user
         # Maybe we can match by username?
         _author = author_name(author)
         user = cls.get_by_username(_author, case_insensitive=True)
         if user is not None:
             return user
 
     def update_lastlogin(self):
         """Update user lastlogin"""
         self.last_login = datetime.datetime.now()
         log.debug('updated user %s lastlogin', self.username)
 
     @classmethod
     def get_first_admin(cls):
         user = User.query().filter(User.admin == True).first()
         if user is None:
             raise Exception('Missing administrative account!')
         return user
 
     @classmethod
     def get_default_user(cls):
         user = User.get_by_username(User.DEFAULT_USER_NAME)
         if user is None:
             raise Exception('Missing default account!')
         return user
 
     def get_api_data(self, details=False):
         """
         Common function for generating user related data for API
         """
         user = self
         data = dict(
             user_id=user.user_id,
             username=user.username,
             firstname=user.name,
             lastname=user.lastname,
             email=user.email,
             emails=user.emails,
             active=user.active,
             admin=user.admin,
         )
         if details:
             data.update(dict(
                 extern_type=user.extern_type,
                 extern_name=user.extern_name,
                 api_key=user.api_key,
                 api_keys=user.api_keys,
                 last_login=user.last_login,
                 ip_addresses=user.ip_addresses
                 ))
         return data
 
     def __json__(self):
         data = dict(
             full_name=self.full_name,
             full_name_or_username=self.full_name_or_username,
             short_contact=self.short_contact,
             full_contact=self.full_contact
         )
         data.update(self.get_api_data())
         return data
 
 
 class UserApiKeys(meta.Base, BaseDbModel):
     __tablename__ = 'user_api_keys'
     __table_args__ = (
         Index('uak_api_key_idx', 'api_key'),
         Index('uak_api_key_expires_idx', 'api_key', 'expires'),
         _table_args_default_dict,
     )
 
     user_api_key_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     api_key = Column(String(255), nullable=False, unique=True)
     description = Column(UnicodeText(), nullable=False)
     expires = Column(Float(53), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     user = relationship('User')
 
     @hybrid_property
     def is_expired(self):
         return (self.expires != -1) & (time.time() > self.expires)
 
 
 class UserEmailMap(meta.Base, BaseDbModel):
     __tablename__ = 'user_email_map'
     __table_args__ = (
         Index('uem_email_idx', 'email'),
         _table_args_default_dict,
     )
 
     email_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     _email = Column("email", String(255), nullable=False, unique=True)
     user = relationship('User')
 
     @validates('_email')
     def validate_email(self, key, email):
         # check if this email is not main one
         main_email = meta.Session().query(User).filter(User.email == email).scalar()
         if main_email is not None:
             raise AttributeError('email %s is present is user table' % email)
         return email
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
 
 class UserIpMap(meta.Base, BaseDbModel):
     __tablename__ = 'user_ip_map'
     __table_args__ = (
         UniqueConstraint('user_id', 'ip_addr'),
         _table_args_default_dict,
     )
 
     ip_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     ip_addr = Column(String(255), nullable=False)
     active = Column(Boolean(), nullable=False, default=True)
     user = relationship('User')
 
     @classmethod
     def _get_ip_range(cls, ip_addr):
         net = ipaddr.IPNetwork(address=ip_addr)
         return [str(net.network), str(net.broadcast)]
 
     def __json__(self):
         return dict(
           ip_addr=self.ip_addr,
           ip_range=self._get_ip_range(self.ip_addr)
         )
 
     def __repr__(self):
         return "<%s %s: %s>" % (self.__class__.__name__, self.user_id, self.ip_addr)
 
 
 class UserLog(meta.Base, BaseDbModel):
     __tablename__ = 'user_logs'
     __table_args__ = (
         _table_args_default_dict,
     )
 
     user_log_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     username = Column(String(255), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     repository_name = Column(Unicode(255), nullable=False)
     user_ip = Column(String(255), nullable=True)
     action = Column(UnicodeText(), nullable=False)
     action_date = Column(DateTime(timezone=False), nullable=False)
 
     def __repr__(self):
         return "<%s %r: %r>" % (self.__class__.__name__,
                                   self.repository_name,
                                   self.action)
 
     @property
     def action_as_day(self):
         return datetime.date(*self.action_date.timetuple()[:3])
 
     user = relationship('User')
     repository = relationship('Repository', cascade='')
 
 
 class UserGroup(meta.Base, BaseDbModel):
     __tablename__ = 'users_groups'
     __table_args__ = (
         _table_args_default_dict,
     )
 
     users_group_id = Column(Integer(), primary_key=True)
     users_group_name = Column(Unicode(255), nullable=False, unique=True)
     user_group_description = Column(Unicode(10000), nullable=True) # FIXME: not nullable?
     users_group_active = Column(Boolean(), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _group_data = Column("group_data", LargeBinary(), nullable=True)  # JSON data # FIXME: not nullable?
 
     members = relationship('UserGroupMember', cascade="all, delete-orphan")
     users_group_to_perm = relationship('UserGroupToPerm', cascade='all')
     users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all')
     user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all')
 
     owner = relationship('User')
 
     @hybrid_property
     def group_data(self):
         if not self._group_data:
             return {}
 
         try:
             return ext_json.loads(self._group_data)
         except TypeError:
             return {}
 
     @group_data.setter
     def group_data(self, val):
         try:
             self._group_data = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
 
     def __repr__(self):
         return "<%s %s: %r>" % (self.__class__.__name__,
                                   self.users_group_id,
                                   self.users_group_name)
 
     @classmethod
     def guess_instance(cls, value):
         return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(sqlalchemy.func.lower(cls.users_group_name) == sqlalchemy.func.lower(group_name))
         else:
             q = cls.query().filter(cls.users_group_name == group_name)
         return q.scalar()
 
     @classmethod
     def get(cls, user_group_id):
         user_group = cls.query()
         return user_group.get(user_group_id)
 
     def get_api_data(self, with_members=True):
         user_group = self
 
         data = dict(
             users_group_id=user_group.users_group_id,
             group_name=user_group.users_group_name,
             group_description=user_group.user_group_description,
             active=user_group.users_group_active,
             owner=user_group.owner.username,
         )
         if with_members:
             data['members'] = [
                 ugm.user.get_api_data()
                 for ugm in user_group.members
             ]
 
         return data
 
 
 class UserGroupMember(meta.Base, BaseDbModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = (
         _table_args_default_dict,
     )
 
     users_group_member_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
 
     user = relationship('User')
     users_group = relationship('UserGroup')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
 
 class RepositoryField(meta.Base, BaseDbModel):
     __tablename__ = 'repositories_fields'
     __table_args__ = (
         UniqueConstraint('repository_id', 'field_key'),  # no-multi field
         _table_args_default_dict,
     )
 
     PREFIX = 'ex_'  # prefix used in form to not conflict with already existing fields
 
     repo_field_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     field_key = Column(String(250), nullable=False)
     field_label = Column(String(1024), nullable=False)
     field_value = Column(String(10000), nullable=False)
     field_desc = Column(String(1024), nullable=False)
     field_type = Column(String(255), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     repository = relationship('Repository')
 
     @property
     def field_key_prefixed(self):
         return 'ex_%s' % self.field_key
 
     @classmethod
     def un_prefix_key(cls, key):
         if key.startswith(cls.PREFIX):
             return key[len(cls.PREFIX):]
         return key
 
     @classmethod
     def get_by_key_name(cls, key, repo):
         row = cls.query() \
                 .filter(cls.repository == repo) \
                 .filter(cls.field_key == key).scalar()
         return row
 
 
 class Repository(meta.Base, BaseDbModel):
     __tablename__ = 'repositories'
     __table_args__ = (
         Index('r_repo_name_idx', 'repo_name'),
         _table_args_default_dict,
     )
 
     DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
     DEFAULT_CLONE_SSH = 'ssh://{system_user}@{hostname}/{repo}'
 
     STATE_CREATED = 'repo_state_created'
     STATE_PENDING = 'repo_state_pending'
     STATE_ERROR = 'repo_state_error'
 
     repo_id = Column(Integer(), primary_key=True)
     repo_name = Column(Unicode(255), nullable=False, unique=True)
     repo_state = Column(String(255), nullable=False)
 
     clone_uri = Column(String(255), nullable=True) # FIXME: not nullable?
     repo_type = Column(String(255), nullable=False) # 'hg' or 'git'
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     private = Column(Boolean(), nullable=False)
     enable_statistics = Column("statistics", Boolean(), nullable=False, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=False, default=True)
     description = Column(Unicode(10000), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _landing_revision = Column("landing_revision", String(255), nullable=False)
     _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True) # JSON data # FIXME: not nullable?
 
     fork_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True)
 
     owner = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing',
                              primaryjoin='UserFollowing.follows_repository_id==Repository.repo_id',
                              cascade='all')
     extra_fields = relationship('RepositoryField',
                                 cascade="all, delete-orphan")
 
     logs = relationship('UserLog')
     comments = relationship('ChangesetComment', cascade="all, delete-orphan")
 
     pull_requests_org = relationship('PullRequest',
                     primaryjoin='PullRequest.org_repo_id==Repository.repo_id',
                     cascade="all, delete-orphan")
 
     pull_requests_other = relationship('PullRequest',
                     primaryjoin='PullRequest.other_repo_id==Repository.repo_id',
                     cascade="all, delete-orphan")
 
     def __repr__(self):
         return "<%s %s: %r>" % (self.__class__.__name__,
                                   self.repo_id, self.repo_name)
 
     @hybrid_property
     def landing_rev(self):
         # always should return [rev_type, rev]
         if self._landing_revision:
             _rev_info = self._landing_revision.split(':')
             if len(_rev_info) < 2:
                 _rev_info.insert(0, 'rev')
             return [_rev_info[0], _rev_info[1]]
         return [None, None]
 
     @landing_rev.setter
     def landing_rev(self, val):
         if ':' not in val:
             raise ValueError('value must be delimited with `:` and consist '
                              'of <rev_type>:<rev>, got %s instead' % val)
         self._landing_revision = val
 
     @hybrid_property
     def changeset_cache(self):
         try:
             cs_cache = ext_json.loads(self._changeset_cache) # might raise on bad data
             cs_cache['raw_id'] # verify data, raise exception on error
             return cs_cache
         except (TypeError, KeyError, ValueError):
             return EmptyChangeset().__json__()
 
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
             self._changeset_cache = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
 
     @classmethod
     def query(cls, sorted=False):
         """Add Repository-specific helpers for common query constructs.
 
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(Repository, cls).query()
 
         if sorted:
             q = q.order_by(sqlalchemy.func.lower(Repository.repo_name))
 
         return q
 
     @classmethod
     def normalize_repo_name(cls, repo_name):
         """
         Normalizes os specific repo_name to the format internally stored inside
         database using URL_SEP
 
         :param cls:
         :param repo_name:
         """
         return kallithea.URL_SEP.join(repo_name.split(os.sep))
 
     @classmethod
     def guess_instance(cls, value):
         return super(Repository, cls).guess_instance(value, Repository.get_by_repo_name)
 
     @classmethod
     def get_by_repo_name(cls, repo_name, case_insensitive=False):
         """Get the repo, defaulting to database case sensitivity.
         case_insensitive will be slower and should only be specified if necessary."""
         if case_insensitive:
             q = meta.Session().query(cls).filter(sqlalchemy.func.lower(cls.repo_name) == sqlalchemy.func.lower(repo_name))
         else:
             q = meta.Session().query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork)) \
                 .options(joinedload(Repository.owner)) \
                 .options(joinedload(Repository.group))
         return q.scalar()
 
     @classmethod
     def get_by_full_path(cls, repo_full_path):
         base_full_path = os.path.realpath(kallithea.CONFIG['base_path'])
         repo_full_path = os.path.realpath(repo_full_path)
         assert repo_full_path.startswith(base_full_path + os.path.sep)
         repo_name = repo_full_path[len(base_full_path) + 1:]
         repo_name = cls.normalize_repo_name(repo_name)
         return cls.get_by_repo_name(repo_name.strip(kallithea.URL_SEP))
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @property
     def forks(self):
         """
         Return forks of this repo
         """
         return Repository.get_repo_forks(self.repo_id)
 
     @property
     def parent(self):
         """
         Returns fork parent
         """
         return self.fork
 
     @property
     def just_name(self):
         return self.repo_name.split(kallithea.URL_SEP)[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         group = self.group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
 
     @property
     def repo_full_path(self):
         """
         Returns base full path for the repository - where it actually
         exists on a filesystem.
         """
         p = [kallithea.CONFIG['base_path']]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(kallithea.URL_SEP)
         return os.path.join(*p)
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return kallithea.URL_SEP.join(path_prefix + [repo_name])
 
     @property
     def _ui(self):
         """
         Creates an db based ui object for this repository
         """
         from kallithea.lib.utils import make_ui
         return make_ui()
 
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
 
         :param cls:
         :param repo_name:
         """
         from kallithea.lib.utils import is_valid_repo
 
         return is_valid_repo(repo_name, kallithea.CONFIG['base_path'])
 
     def get_api_data(self, with_revision_names=False,
                            with_pullrequests=False):
         """
         Common function for generating repo api data.
         Optionally, also return tags, branches, bookmarks and PRs.
         """
         repo = self
         data = dict(
             repo_id=repo.repo_id,
             repo_name=repo.repo_name,
             repo_type=repo.repo_type,
             clone_uri=repo.clone_uri,
             private=repo.private,
             created_on=repo.created_on,
             description=repo.description,
             landing_rev=repo.landing_rev,
             owner=repo.owner.username,
             fork_of=repo.fork.repo_name if repo.fork else None,
             enable_statistics=repo.enable_statistics,
             enable_downloads=repo.enable_downloads,
             last_changeset=repo.changeset_cache,
         )
         if with_revision_names:
             scm_repo = repo.scm_instance_no_cache()
             data.update(dict(
                 tags=scm_repo.tags,
                 branches=scm_repo.branches,
                 bookmarks=scm_repo.bookmarks,
             ))
         if with_pullrequests:
             data['pull_requests'] = repo.pull_requests_other
         settings = Setting.get_app_settings()
         repository_fields = asbool(settings.get('repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
 
         return data
 
     @property
     def last_db_change(self):
         return self.updated_on
 
     @property
     def clone_uri_hidden(self):
         clone_uri = self.clone_uri
         if clone_uri:
-            import urlobject
             url_obj = urlobject.URLObject(self.clone_uri)
             if url_obj.password:
                 clone_uri = url_obj.with_password('*****')
         return clone_uri
 
     def clone_url(self, clone_uri_tmpl, with_id=False, username=None):
         if '{repo}' not in clone_uri_tmpl and '_{repoid}' not in clone_uri_tmpl:
             log.error("Configured clone_uri_tmpl %r has no '{repo}' or '_{repoid}' and cannot toggle to use repo id URLs", clone_uri_tmpl)
         elif with_id:
             clone_uri_tmpl = clone_uri_tmpl.replace('{repo}', '_{repoid}')
         else:
             clone_uri_tmpl = clone_uri_tmpl.replace('_{repoid}', '{repo}')
 
         prefix_url = webutils.canonical_url('home')
 
         return get_clone_url(clone_uri_tmpl=clone_uri_tmpl,
                              prefix_url=prefix_url,
                              repo_name=self.repo_name,
                              repo_id=self.repo_id,
                              username=username)
 
     def set_state(self, state):
         self.repo_state = state
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_changeset(self, rev=None):
         return get_changeset_safe(self.scm_instance, rev)
 
     def get_landing_changeset(self):
         """
         Returns landing changeset, or if that doesn't exist returns the tip
         """
         _rev_type, _rev = self.landing_rev
         cs = self.get_changeset(_rev)
         if isinstance(cs, EmptyChangeset):
             return self.get_changeset()
         return cs
 
     def update_changeset_cache(self, cs_cache=None):
         """
         Update cache of last changeset for repository, keys should be::
 
             short_id
             raw_id
             revision
             message
             date
             author
 
         :param cs_cache:
         """
-        from kallithea.lib.vcs.backends.base import BaseChangeset
         if cs_cache is None:
             cs_cache = EmptyChangeset()
             # use no-cache version here
             scm_repo = self.scm_instance_no_cache()
             if scm_repo:
                 cs_cache = scm_repo.get_changeset()
 
         if isinstance(cs_cache, BaseChangeset):
             cs_cache = cs_cache.__json__()
 
         if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):
             _default = datetime.datetime.fromtimestamp(0)
             last_change = cs_cache.get('date') or _default
             log.debug('updated repo %s with new cs cache %s',
                       self.repo_name, cs_cache)
             self.updated_on = last_change
             self.changeset_cache = cs_cache
             meta.Session().commit()
         else:
             log.debug('changeset_cache for %s already up to date with %s',
                       self.repo_name, cs_cache['raw_id'])
 
     @property
     def tip(self):
         return self.get_changeset('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance.last_change
 
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
 
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query() \
             .filter(ChangesetComment.repo == self)
         if revisions is not None:
             if not revisions:
                 return {} # don't use sql 'in' on empty set
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = collections.defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
 
     def statuses(self, revisions):
         """
         Returns statuses for this repository.
         PRs without any votes do _not_ show up as unreviewed.
 
         :param revisions: list of revisions to get statuses for
         """
         if not revisions:
             return {}
 
         statuses = ChangesetStatus.query() \
             .filter(ChangesetStatus.repo == self) \
             .filter(ChangesetStatus.version == 0) \
             .filter(ChangesetStatus.revision.in_(revisions))
 
         grouped = {}
         for stat in statuses.all():
             pr_id = pr_nice_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_nice_id = PullRequest.make_nice_id(pr_id)
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo, pr_nice_id,
                                       stat.author]
         return grouped
 
     def _repo_size(self):
         log.debug('calculating repository size...')
         return webutils.format_byte_size(self.scm_instance.size)
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     def set_invalidate(self):
         """
         Flush SA session caches of instances of on disk repo.
         """
         try:
             del self._scm_instance
         except AttributeError:
             pass
 
     _scm_instance = None  # caching inside lifetime of SA session
 
     @property
     def scm_instance(self):
         if self._scm_instance is None:
             return self.scm_instance_no_cache()  # will populate self._scm_instance
         return self._scm_instance
 
     def scm_instance_no_cache(self):
         repo_full_path = self.repo_full_path
         alias = get_scm(repo_full_path)[0]
         log.debug('Creating instance of %s repository from %s',
                   alias, self.repo_full_path)
         backend = get_backend(alias)
 
         if alias == 'hg':
             self._scm_instance = backend(repo_full_path, create=False, baseui=self._ui)
         else:
             self._scm_instance = backend(repo_full_path, create=False)
 
         return self._scm_instance
 
     def __json__(self):
         return dict(
             repo_id=self.repo_id,
             repo_name=self.repo_name,
             landing_rev=self.landing_rev,
         )
 
 
 class RepoGroup(meta.Base, BaseDbModel):
     __tablename__ = 'groups'
     __table_args__ = (
         _table_args_default_dict,
     )
 
     SEP = ' &raquo; '
 
     group_id = Column(Integer(), primary_key=True)
     group_name = Column(Unicode(255), nullable=False, unique=True) # full path
     parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)
     group_description = Column(Unicode(10000), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     owner = relationship('User')
 
     @classmethod
     def query(cls, sorted=False):
         """Add RepoGroup-specific helpers for common query constructs.
 
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(RepoGroup, cls).query()
 
         if sorted:
             q = q.order_by(sqlalchemy.func.lower(RepoGroup.group_name))
 
         return q
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__(self):
         return "<%s %s: %s>" % (self.__class__.__name__,
                                 self.group_id, self.group_name)
 
     @classmethod
     def _generate_choice(cls, repo_group):
         """Return tuple with group_id and name as html literal"""
         if repo_group is None:
             return (-1, '-- %s --' % _('top level'))
         return repo_group.group_id, webutils.literal(cls.SEP.join(repo_group.full_path_splitted))
 
     @classmethod
     def groups_choices(cls, groups):
         """Return tuples with group_id and name as html literal."""
         return sorted((cls._generate_choice(g) for g in groups),
                       key=lambda c: c[1].split(cls.SEP))
 
     @classmethod
     def guess_instance(cls, value):
         return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, case_insensitive=False):
         group_name = group_name.rstrip('/')
         if case_insensitive:
             gr = cls.query() \
                 .filter(sqlalchemy.func.lower(cls.group_name) == sqlalchemy.func.lower(group_name))
         else:
             gr = cls.query() \
                 .filter(cls.group_name == group_name)
         return gr.scalar()
 
     @property
     def parents(self):
         groups = []
         group = self.parent_group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
 
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(kallithea.URL_SEP)[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(kallithea.URL_SEP)
 
     @property
     def repositories(self):
         return Repository.query(sorted=True).filter_by(group=self)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
     def _recursive_objects(self, include_repos=True):
         all_ = []
 
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     all_.append(gr)
                     _get_members(gr)
 
         _get_members(self)
         return [self] + all_
 
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
 
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return kallithea.URL_SEP.join(path_prefix + [group_name])
 
     def get_api_data(self):
         """
         Common function for generating api data
 
         """
         group = self
         data = dict(
             group_id=group.group_id,
             group_name=group.group_name,
             group_description=group.group_description,
             parent_group=group.parent_group.group_name if group.parent_group else None,
             repositories=[x.repo_name for x in group.repositories],
             owner=group.owner.username
         )
         return data
 
 
 class Permission(meta.Base, BaseDbModel):
     __tablename__ = 'permissions'
     __table_args__ = (
         Index('p_perm_name_idx', 'permission_name'),
         _table_args_default_dict,
     )
 
     PERMS = (
         ('hg.admin', _('Kallithea Administrator')),
 
         ('repository.none', _('Default user has no access to new repositories')),
         ('repository.read', _('Default user has read access to new repositories')),
         ('repository.write', _('Default user has write access to new repositories')),
         ('repository.admin', _('Default user has admin access to new repositories')),
 
         ('group.none', _('Default user has no access to new repository groups')),
         ('group.read', _('Default user has read access to new repository groups')),
         ('group.write', _('Default user has write access to new repository groups')),
         ('group.admin', _('Default user has admin access to new repository groups')),
 
         ('usergroup.none', _('Default user has no access to new user groups')),
         ('usergroup.read', _('Default user has read access to new user groups')),
         ('usergroup.write', _('Default user has write access to new user groups')),
         ('usergroup.admin', _('Default user has admin access to new user groups')),
 
         ('hg.usergroup.create.false', _('Only admins can create user groups')),
         ('hg.usergroup.create.true', _('Non-admins can create user groups')),
 
         ('hg.create.none', _('Only admins can create top level repositories')),
         ('hg.create.repository', _('Non-admins can create top level repositories')),
 
         ('hg.fork.none', _('Only admins can fork repositories')),
         ('hg.fork.repository', _('Non-admins can fork repositories')),
 
         ('hg.register.none', _('Registration disabled')),
         ('hg.register.manual_activate', _('User registration with manual account activation')),
         ('hg.register.auto_activate', _('User registration with automatic account activation')),
 
         ('hg.extern_activate.manual', _('Manual activation of external account')),
         ('hg.extern_activate.auto', _('Automatic activation of external account')),
     )
 
     # definition of system default permissions for DEFAULT user
     DEFAULT_USER_PERMISSIONS = (
         'repository.read',
         'group.read',
         'usergroup.read',
         'hg.create.repository',
         'hg.fork.repository',
         'hg.register.manual_activate',
         'hg.extern_activate.auto',
     )
 
     # defines which permissions are more important higher the more important
     # Weight defines which permissions are more important.
     # The higher number the more important.
     PERM_WEIGHTS = {
         'repository.none': 0,
         'repository.read': 1,
         'repository.write': 3,
         'repository.admin': 4,
 
         'group.none': 0,
         'group.read': 1,
         'group.write': 3,
         'group.admin': 4,
 
         'usergroup.none': 0,
         'usergroup.read': 1,
         'usergroup.write': 3,
         'usergroup.admin': 4,
 
         'hg.usergroup.create.false': 0,
         'hg.usergroup.create.true': 1,
 
         'hg.fork.none': 0,
         'hg.fork.repository': 1,
 
         'hg.create.none': 0,
         'hg.create.repository': 1,
 
         'hg.register.none': 0,
         'hg.register.manual_activate': 1,
         'hg.register.auto_activate': 2,
 
         'hg.extern_activate.manual': 0,
         'hg.extern_activate.auto': 1,
     }
 
     permission_id = Column(Integer(), primary_key=True)
     permission_name = Column(String(255), nullable=False)
 
     def __repr__(self):
         return "<%s %s: %r>" % (
             self.__class__.__name__, self.permission_id, self.permission_name
         )
 
     @classmethod
     def guess_instance(cls, value):
         return super(Permission, cls).guess_instance(value, Permission.get_by_key)
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
     @classmethod
     def get_default_perms(cls, default_user_id):
         q = meta.Session().query(UserRepoToPerm) \
          .options(joinedload(UserRepoToPerm.repository)) \
          .options(joinedload(UserRepoToPerm.permission)) \
          .filter(UserRepoToPerm.user_id == default_user_id)
 
         return q.all()
 
     @classmethod
     def get_default_group_perms(cls, default_user_id):
         q = meta.Session().query(UserRepoGroupToPerm) \
          .options(joinedload(UserRepoGroupToPerm.group)) \
          .options(joinedload(UserRepoGroupToPerm.permission)) \
          .filter(UserRepoGroupToPerm.user_id == default_user_id)
 
         return q.all()
 
     @classmethod
     def get_default_user_group_perms(cls, default_user_id):
         q = meta.Session().query(UserUserGroupToPerm) \
          .options(joinedload(UserUserGroupToPerm.user_group)) \
          .options(joinedload(UserUserGroupToPerm.permission)) \
          .filter(UserUserGroupToPerm.user_id == default_user_id)
 
         return q.all()
 
 
 class UserRepoToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'repository_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     repo_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
 
     user = relationship('User')
     repository = relationship('Repository')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, repository, permission):
         n = cls()
         n.user = user
         n.repository = repository
         n.permission = permission
         meta.Session().add(n)
         return n
 
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.user, self.repository, self.permission)
 
 
 class UserUserGroupToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'user_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'user_group_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     user_user_group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
 
     user = relationship('User')
     user_group = relationship('UserGroup')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, user_group, permission):
         n = cls()
         n.user = user
         n.user_group = user_group
         n.permission = permission
         meta.Session().add(n)
         return n
 
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.user, self.user_group, self.permission)
 
 
 class UserToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     user_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
 
     user = relationship('User')
     permission = relationship('Permission')
 
     def __repr__(self):
         return '<%s %s: %s>' % (
             self.__class__.__name__, self.user, self.permission)
 
 
 class UserGroupRepoToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (
         UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
     @classmethod
     def create(cls, users_group, repository, permission):
         n = cls()
         n.users_group = users_group
         n.repository = repository
         n.permission = permission
         meta.Session().add(n)
         return n
 
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.users_group, self.repository, self.permission)
 
 
 class UserGroupUserGroupToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'user_group_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     user_group_user_group_to_perm_id = Column(Integer(), primary_key=True)
     target_user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
 
     target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id')
     user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, target_user_group, user_group, permission):
         n = cls()
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         meta.Session().add(n)
         return n
 
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.user_group, self.target_user_group, self.permission)
 
 
 class UserGroupToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         _table_args_default_dict,
     )
 
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 
 
 class UserRepoGroupToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         _table_args_default_dict,
     )
 
     group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
 
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, repository_group, permission):
         n = cls()
         n.user = user
         n.group = repository_group
         n.permission = permission
         meta.Session().add(n)
         return n
 
 
 class UserGroupRepoGroupToPerm(meta.Base, BaseDbModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         _table_args_default_dict,
     )
 
     users_group_repo_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
 
     @classmethod
     def create(cls, user_group, repository_group, permission):
         n = cls()
         n.users_group = user_group
         n.group = repository_group
         n.permission = permission
         meta.Session().add(n)
         return n
 
 
 class Statistics(meta.Base, BaseDbModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          _table_args_default_dict,
     )
 
     stat_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True)
     stat_on_revision = Column(Integer(), nullable=False)
     commit_activity = Column(LargeBinary(1000000), nullable=False) # JSON data
     commit_activity_combined = Column(LargeBinary(), nullable=False) # JSON data
     languages = Column(LargeBinary(1000000), nullable=False) # JSON data
 
     repository = relationship('Repository', single_parent=True)
 
 
 class UserFollowing(meta.Base, BaseDbModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id', name='uq_user_followings_user_repo'),
         UniqueConstraint('user_id', 'follows_user_id', name='uq_user_followings_user_user'),
         _table_args_default_dict,
     )
 
     user_following_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     follows_repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     follows_user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     follows_from = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by=lambda: sqlalchemy.func.lower(Repository.repo_name))
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repository_id == repo_id)
 
 
 class ChangesetComment(meta.Base, BaseDbModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         Index('cc_pull_request_id_idx', 'pull_request_id'),
         _table_args_default_dict,
     )
 
     comment_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     line_no = Column(Unicode(10), nullable=True)
     f_path = Column(Unicode(1000), nullable=True)
     author_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column(UnicodeText(), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     author = relationship('User')
     repo = relationship('Repository')
     # status_change is frequently used directly in templates - make it a lazy
     # join to avoid fetching each related ChangesetStatus on demand.
     # There will only be one ChangesetStatus referencing each comment so the join will not explode.
     status_change = relationship('ChangesetStatus',
                                  cascade="all, delete-orphan", lazy='joined')
     pull_request = relationship('PullRequest')
 
     def url(self):
         anchor = "comment-%s" % self.comment_id
         if self.revision:
             return webutils.url('changeset_home', repo_name=self.repo.repo_name, revision=self.revision, anchor=anchor)
         elif self.pull_request_id is not None:
             return self.pull_request.url(anchor=anchor)
 
     def __json__(self):
         return dict(
             comment_id=self.comment_id,
             username=self.author.username,
             text=self.text,
         )
 
     def deletable(self):
         return self.created_on > datetime.datetime.now() - datetime.timedelta(minutes=5)
 
 
 class ChangesetStatus(meta.Base, BaseDbModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         Index('cs_pull_request_id_idx', 'pull_request_id'),
         Index('cs_changeset_comment_id_idx', 'changeset_comment_id'),
         Index('cs_pull_request_id_user_id_version_idx', 'pull_request_id', 'user_id', 'version'),
         Index('cs_repo_id_pull_request_id_idx', 'repo_id', 'pull_request_id'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         _table_args_default_dict,
     )
 
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected' # is shown as "Not approved" - TODO: change database content / scheme
     STATUS_UNDER_REVIEW = 'under_review'
 
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not reviewed")),  # (no icon) and default
         (STATUS_UNDER_REVIEW, _("Under review")),
         (STATUS_REJECTED, _("Not approved")),
         (STATUS_APPROVED, _("Approved")),
     ]
     STATUSES_DICT = dict(STATUSES)
 
     changeset_status_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     status = Column(String(128), nullable=False, default=DEFAULT)
     comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)
     modified_at = Column(DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column(Integer(), nullable=False, default=0)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
 
     author = relationship('User')
     repo = relationship('Repository')
     comment = relationship('ChangesetComment')
     pull_request = relationship('PullRequest')
 
     def __repr__(self):
         return "<%s %r by %r>" % (
             self.__class__.__name__,
             self.status, self.author
         )
 
     @classmethod
     def get_status_lbl(cls, value):
         return cls.STATUSES_DICT.get(value)
 
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
 
     def __json__(self):
         return dict(
             status=self.status,
             modified_at=self.modified_at.replace(microsecond=0),
             reviewer=self.author.username,
             )
 
 
 class PullRequest(meta.Base, BaseDbModel):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         Index('pr_org_repo_id_idx', 'org_repo_id'),
         Index('pr_other_repo_id_idx', 'other_repo_id'),
         _table_args_default_dict,
     )
 
     # values for .status
     STATUS_NEW = 'new'
     STATUS_CLOSED = 'closed'
 
     pull_request_id = Column(Integer(), primary_key=True)
     title = Column(Unicode(255), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     status = Column(Unicode(255), nullable=False, default=STATUS_NEW) # only for closedness, not approve/reject/etc
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     _revisions = Column('revisions', UnicodeText(), nullable=False)
     org_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     org_ref = Column(Unicode(255), nullable=False)
     other_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     other_ref = Column(Unicode(255), nullable=False)
 
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':')
 
     @revisions.setter
     def revisions(self, val):
         self._revisions = ':'.join(val)
 
     @property
     def org_ref_parts(self):
         return self.org_ref.split(':')
 
     @property
     def other_ref_parts(self):
         return self.other_ref.split(':')
 
     owner = relationship('User')
     reviewers = relationship('PullRequestReviewer',
                              cascade="all, delete-orphan")
     org_repo = relationship('Repository', primaryjoin='PullRequest.org_repo_id==Repository.repo_id')
     other_repo = relationship('Repository', primaryjoin='PullRequest.other_repo_id==Repository.repo_id')
     statuses = relationship('ChangesetStatus', order_by='ChangesetStatus.changeset_status_id')
     comments = relationship('ChangesetComment', order_by='ChangesetComment.comment_id',
                              cascade="all, delete-orphan")
 
     @classmethod
     def query(cls, reviewer_id=None, include_closed=True, sorted=False):
         """Add PullRequest-specific helpers for common query constructs.
 
         reviewer_id: only PRs with the specified user added as reviewer.
 
         include_closed: if False, do not include closed PRs.
 
         sorted: if True, apply the default ordering (newest first).
         """
         q = super(PullRequest, cls).query()
 
         if reviewer_id is not None:
             q = q.join(PullRequestReviewer).filter(PullRequestReviewer.user_id == reviewer_id)
 
         if not include_closed:
             q = q.filter(PullRequest.status != PullRequest.STATUS_CLOSED)
 
         if sorted:
             q = q.order_by(PullRequest.created_on.desc())
 
         return q
 
     def get_reviewer_users(self):
         """Like .reviewers, but actually returning the users"""
         return User.query() \
             .join(PullRequestReviewer) \
             .filter(PullRequestReviewer.pull_request == self) \
             .order_by(PullRequestReviewer.pull_request_reviewers_id) \
             .all()
 
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
 
     def user_review_status(self, user_id):
         """Return the user's latest status votes on PR"""
         # note: no filtering on repo - that would be redundant
         status = ChangesetStatus.query() \
             .filter(ChangesetStatus.pull_request == self) \
             .filter(ChangesetStatus.user_id == user_id) \
             .order_by(ChangesetStatus.version) \
             .first()
         return str(status.status) if status else ''
 
     @classmethod
     def make_nice_id(cls, pull_request_id):
         '''Return pull request id nicely formatted for displaying'''
         return '#%s' % pull_request_id
 
     def nice_id(self):
         '''Return the id of this pull request, nicely formatted for displaying'''
         return self.make_nice_id(self.pull_request_id)
 
     def get_api_data(self):
         return self.__json__()
 
     def __json__(self):
         clone_uri_tmpl = kallithea.CONFIG.get('clone_uri_tmpl') or Repository.DEFAULT_CLONE_URI
         return dict(
             pull_request_id=self.pull_request_id,
             url=self.url(),
             reviewers=self.reviewers,
             revisions=self.revisions,
             owner=self.owner.username,
             title=self.title,
             description=self.description,
             org_repo_url=self.org_repo.clone_url(clone_uri_tmpl=clone_uri_tmpl),
             org_ref_parts=self.org_ref_parts,
             other_ref_parts=self.other_ref_parts,
             status=self.status,
             comments=self.comments,
             statuses=self.statuses,
             created_on=self.created_on.replace(microsecond=0),
             updated_on=self.updated_on.replace(microsecond=0),
         )
 
     def url(self, **kwargs):
         canonical = kwargs.pop('canonical', None)
         b = self.org_ref_parts[1]
         if b != self.other_ref_parts[1]:
             s = '/_/' + b
         else:
             s = '/_/' + self.title
         kwargs['extra'] = urlreadable(s)
         if canonical:
             return webutils.canonical_url('pullrequest_show', repo_name=self.other_repo.repo_name,
                                    pull_request_id=self.pull_request_id, **kwargs)
         return webutils.url('pullrequest_show', repo_name=self.other_repo.repo_name,
                      pull_request_id=self.pull_request_id, **kwargs)
 
 
 class PullRequestReviewer(meta.Base, BaseDbModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         Index('pull_request_reviewers_user_id_idx', 'user_id'),
         UniqueConstraint('pull_request_id', 'user_id'),
         _table_args_default_dict,
     )
 
     def __init__(self, user=None, pull_request=None):
         self.user = user
         self.pull_request = pull_request
 
     pull_request_reviewers_id = Column('pull_requests_reviewers_id', Integer(), primary_key=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
 
     user = relationship('User')
     pull_request = relationship('PullRequest')
 
     def __json__(self):
         return dict(
             username=self.user.username if self.user else None,
         )
 
 
 class Notification(object):
     __tablename__ = 'notifications'
 
 class UserNotification(object):
     __tablename__ = 'user_to_notification'
 
 
 class Gist(meta.Base, BaseDbModel):
     __tablename__ = 'gists'
     __table_args__ = (
         Index('g_gist_access_id_idx', 'gist_access_id'),
         Index('g_created_on_idx', 'created_on'),
         _table_args_default_dict,
     )
 
     GIST_STORE_LOC = '.rc_gist_store'
     GIST_METADATA_FILE = '.rc_gist_metadata'
 
     GIST_PUBLIC = 'public'
     GIST_PRIVATE = 'private'
     DEFAULT_FILENAME = 'gistfile1.txt'
 
     gist_id = Column(Integer(), primary_key=True)
     gist_access_id = Column(Unicode(250), nullable=False)
     gist_description = Column(UnicodeText(), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     gist_expires = Column(Float(53), nullable=False)
     gist_type = Column(Unicode(128), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     owner = relationship('User')
 
     @hybrid_property
     def is_expired(self):
         return (self.gist_expires != -1) & (time.time() > self.gist_expires)
 
     def __repr__(self):
         return "<%s %s %s>" % (
             self.__class__.__name__,
             self.gist_type, self.gist_access_id)
 
     @classmethod
     def guess_instance(cls, value):
         return super(Gist, cls).guess_instance(value, Gist.get_by_access_id)
 
     @classmethod
     def get_or_404(cls, id_):
         res = cls.query().filter(cls.gist_access_id == id_).scalar()
         if res is None:
             raise HTTPNotFound
         return res
 
     @classmethod
     def get_by_access_id(cls, gist_access_id):
         return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
 
     def gist_url(self):
         alias_url = kallithea.CONFIG.get('gist_alias_url')
         if alias_url:
             return alias_url.replace('{gistid}', self.gist_access_id)
 
         return webutils.canonical_url('gist', gist_id=self.gist_access_id)
 
     def get_api_data(self):
         """
         Common function for generating gist related data for API
         """
         gist = self
         data = dict(
             gist_id=gist.gist_id,
             type=gist.gist_type,
             access_id=gist.gist_access_id,
             description=gist.gist_description,
             url=gist.gist_url(),
             expires=gist.gist_expires,
             created_on=gist.created_on,
         )
         return data
 
     def __json__(self):
         data = dict(
         )
         data.update(self.get_api_data())
         return data
 
     ## SCM functions
 
     @property
     def scm_instance(self):
         gist_base_path = os.path.join(kallithea.CONFIG['base_path'], self.GIST_STORE_LOC)
         return get_repo(os.path.join(gist_base_path, self.gist_access_id))
 
 
 class UserSshKeys(meta.Base, BaseDbModel):
     __tablename__ = 'user_ssh_keys'
     __table_args__ = (
         Index('usk_fingerprint_idx', 'fingerprint'),
         _table_args_default_dict
     )
     __mapper_args__ = {}
 
     user_ssh_key_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     _public_key = Column('public_key', UnicodeText(), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     fingerprint = Column(String(255), nullable=False, unique=True)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     last_seen = Column(DateTime(timezone=False), nullable=True)
 
     user = relationship('User')
 
     @property
     def public_key(self):
         return self._public_key
 
     @public_key.setter
     def public_key(self, full_key):
         """The full public key is too long to be suitable as database key.
         Instead, as a side-effect of setting the public key string, compute the
         fingerprints according to https://tools.ietf.org/html/rfc4716#section-4
         BUT using sha256 instead of md5, similar to 'ssh-keygen -E sha256 -lf
         ~/.ssh/id_rsa.pub' .
         """
         keytype, key_bytes, comment = ssh.parse_pub_key(full_key)
         self._public_key = full_key
         self.fingerprint = base64.b64encode(hashlib.sha256(key_bytes).digest()).replace(b'\n', b'').rstrip(b'=').decode()

kallithea/model/user.py

@@ -1,505 +1,503 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.user
 ~~~~~~~~~~~~~~~~~~~~
 
 users model for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 9, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import hashlib
 import hmac
 import logging
 import time
 import traceback
 
 from sqlalchemy.exc import DatabaseError
 from tg import config
 from tg.i18n import ugettext as _
 
 from kallithea.lib import webutils
 from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException
 from kallithea.lib.utils2 import generate_api_key, get_current_authuser
-from kallithea.model import db, meta
+from kallithea.model import db, forms, meta
 
 
 log = logging.getLogger(__name__)
 
 
 class UserModel(object):
     password_reset_token_lifetime = 86400 # 24 hours
 
     def get(self, user_id):
         user = db.User.query()
         return user.get(user_id)
 
     def get_user(self, user):
         return db.User.guess_instance(user)
 
     def create(self, form_data, cur_user=None):
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
 
         from kallithea.lib.hooks import check_allowed_create_user, log_create_user
         _fd = form_data
         user_data = {
             'username': _fd['username'],
             'password': _fd['password'],
             'email': _fd['email'],
             'firstname': _fd['firstname'],
             'lastname': _fd['lastname'],
             'active': _fd['active'],
             'admin': False
         }
         # raises UserCreationError if it's not allowed
         check_allowed_create_user(user_data, cur_user)
         from kallithea.lib.auth import get_crypt_password
 
         new_user = db.User()
         for k, v in form_data.items():
             if k == 'password':
                 v = get_crypt_password(v)
             if k == 'firstname':
                 k = 'name'
             setattr(new_user, k, v)
 
         new_user.api_key = generate_api_key()
         meta.Session().add(new_user)
         meta.Session().flush() # make database assign new_user.user_id
 
         log_create_user(new_user.get_dict(), cur_user)
         return new_user
 
     def create_or_update(self, username, password, email, firstname='',
                          lastname='', active=True, admin=False,
                          extern_type=None, extern_name=None, cur_user=None):
         """
         Creates a new instance if not found, or updates current one
 
         :param username:
         :param password:
         :param email:
         :param active:
         :param firstname:
         :param lastname:
         :param active:
         :param admin:
         :param extern_name:
         :param extern_type:
         :param cur_user:
         """
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
 
         from kallithea.lib.auth import check_password, get_crypt_password
         from kallithea.lib.hooks import check_allowed_create_user, log_create_user
         user_data = {
             'username': username, 'password': password,
             'email': email, 'firstname': firstname, 'lastname': lastname,
             'active': active, 'admin': admin
         }
         # raises UserCreationError if it's not allowed
         check_allowed_create_user(user_data, cur_user)
 
         log.debug('Checking for %s account in Kallithea database', username)
         user = db.User.get_by_username(username, case_insensitive=True)
         if user is None:
             log.debug('creating new user %s', username)
             new_user = db.User()
             edit = False
         else:
             log.debug('updating user %s', username)
             new_user = user
             edit = True
 
         try:
             new_user.username = username
             new_user.admin = admin
             new_user.email = email
             new_user.active = active
             new_user.extern_name = extern_name
             new_user.extern_type = extern_type
             new_user.name = firstname
             new_user.lastname = lastname
 
             if not edit:
                 new_user.api_key = generate_api_key()
 
             # set password only if creating an user or password is changed
             password_change = new_user.password and \
                 not check_password(password, new_user.password)
             if not edit or password_change:
                 reason = 'new password' if edit else 'new user'
                 log.debug('Updating password reason=>%s', reason)
                 new_user.password = get_crypt_password(password) \
                     if password else ''
 
             if user is None:
                 meta.Session().add(new_user)
                 meta.Session().flush() # make database assign new_user.user_id
 
             if not edit:
                 log_create_user(new_user.get_dict(), cur_user)
 
             return new_user
         except (DatabaseError,):
             log.error(traceback.format_exc())
             raise
 
     def create_registration(self, form_data):
         from kallithea.model import notification
 
         form_data['admin'] = False
         form_data['extern_type'] = db.User.DEFAULT_AUTH_TYPE
         form_data['extern_name'] = ''
         new_user = self.create(form_data)
 
         # notification to admins
         subject = _('New user registration')
         body = (
             'New user registration\n'
             '---------------------\n'
             '- Username: {user.username}\n'
             '- Full Name: {user.full_name}\n'
             '- Email: {user.email}\n'
             ).format(user=new_user)
         edit_url = webutils.canonical_url('edit_user', id=new_user.user_id)
         email_kwargs = {
             'registered_user_url': edit_url,
             'new_username': new_user.username,
             'new_email': new_user.email,
             'new_full_name': new_user.full_name}
         notification.NotificationModel().create(created_by=new_user, subject=subject,
                                    body=body, recipients=None,
                                    type_=notification.NotificationModel.TYPE_REGISTRATION,
                                    email_kwargs=email_kwargs)
 
     def update(self, user_id, form_data, skip_attrs=None):
         from kallithea.lib.auth import get_crypt_password
         skip_attrs = skip_attrs or []
         user = self.get(user_id)
         if user.is_default_user:
             raise DefaultUserException(
                             _("You can't edit this user since it's "
                               "crucial for entire application"))
 
         for k, v in form_data.items():
             if k in skip_attrs:
                 continue
             if k == 'new_password' and v:
                 user.password = get_crypt_password(v)
             else:
                 # old legacy thing orm models store firstname as name,
                 # need proper refactor to username
                 if k == 'firstname':
                     k = 'name'
                 setattr(user, k, v)
 
     def update_user(self, user, **kwargs):
         from kallithea.lib.auth import get_crypt_password
 
         user = db.User.guess_instance(user)
         if user.is_default_user:
             raise DefaultUserException(
                 _("You can't edit this user since it's"
                   " crucial for entire application")
             )
 
         for k, v in kwargs.items():
             if k == 'password' and v:
                 v = get_crypt_password(v)
 
             setattr(user, k, v)
         return user
 
     def delete(self, user, cur_user=None):
         if cur_user is None:
             cur_user = getattr(get_current_authuser(), 'username', None)
         user = db.User.guess_instance(user)
 
         if user.is_default_user:
             raise DefaultUserException(
                 _("You can't remove this user since it is"
                   " crucial for the entire application"))
         if user.repositories:
             repos = [x.repo_name for x in user.repositories]
             raise UserOwnsReposException(
                 _('User "%s" still owns %s repositories and cannot be '
                   'removed. Switch owners or remove those repositories: %s')
                 % (user.username, len(repos), ', '.join(repos)))
         if user.repo_groups:
             repogroups = [x.group_name for x in user.repo_groups]
             raise UserOwnsReposException(_(
                 'User "%s" still owns %s repository groups and cannot be '
                 'removed. Switch owners or remove those repository groups: %s')
                 % (user.username, len(repogroups), ', '.join(repogroups)))
         if user.user_groups:
             usergroups = [x.users_group_name for x in user.user_groups]
             raise UserOwnsReposException(
                 _('User "%s" still owns %s user groups and cannot be '
                   'removed. Switch owners or remove those user groups: %s')
                 % (user.username, len(usergroups), ', '.join(usergroups)))
         meta.Session().delete(user)
 
         from kallithea.lib.hooks import log_delete_user
         log_delete_user(user.get_dict(), cur_user)
 
     def can_change_password(self, user):
         from kallithea.lib import auth_modules
         managed_fields = auth_modules.get_managed_fields(user)
         return 'password' not in managed_fields
 
     def get_reset_password_token(self, user, timestamp, session_id):
         """
         The token is a 40-digit hexstring, calculated as a HMAC-SHA1.
 
         In a traditional HMAC scenario, an attacker is unable to know or
         influence the secret key, but can know or influence the message
         and token. This scenario is slightly different (in particular
         since the message sender is also the message recipient), but
         sufficiently similar to use an HMAC. Benefits compared to a plain
         SHA1 hash includes resistance against a length extension attack.
 
         The HMAC key consists of the following values (known only to the
         server and authorized users):
 
         * per-application secret (the `app_instance_uuid` setting), without
           which an attacker cannot counterfeit tokens
         * hashed user password, invalidating the token upon password change
 
         The HMAC message consists of the following values (potentially known
         to an attacker):
 
         * session ID (the anti-CSRF token), requiring an attacker to have
           access to the browser session in which the token was created
         * numeric user ID, limiting the token to a specific user (yet allowing
           users to be renamed)
         * user email address
         * time of token issue (a Unix timestamp, to enable token expiration)
 
         The key and message values are separated by NUL characters, which are
         guaranteed not to occur in any of the values.
         """
         app_secret = config.get('app_instance_uuid')
         return hmac.new(
             '\0'.join([app_secret, user.password]).encode('utf-8'),
             msg='\0'.join([session_id, str(user.user_id), user.email, str(timestamp)]).encode('utf-8'),
             digestmod=hashlib.sha1,
         ).hexdigest()
 
     def send_reset_password_email(self, data):
         """
         Sends email with a password reset token and link to the password
         reset confirmation page with all information (including the token)
         pre-filled. Also returns URL of that page, only without the token,
         allowing users to copy-paste or manually enter the token from the
         email.
         """
         from kallithea.lib.celerylib import tasks
         from kallithea.model import notification
 
         user_email = data['email']
         user = db.User.get_by_email(user_email)
         timestamp = int(time.time())
         if user is not None:
             if self.can_change_password(user):
                 log.debug('password reset user %s found', user)
                 token = self.get_reset_password_token(user,
                                                       timestamp,
                                                       webutils.session_csrf_secret_token())
                 # URL must be fully qualified; but since the token is locked to
                 # the current browser session, we must provide a URL with the
                 # current scheme and hostname, rather than the canonical_url.
                 link = webutils.url('reset_password_confirmation', qualified=True,
                              email=user_email,
                              timestamp=timestamp,
                              token=token)
             else:
                 log.debug('password reset user %s found but was managed', user)
                 token = link = None
             reg_type = notification.EmailNotificationModel.TYPE_PASSWORD_RESET
             body = notification.EmailNotificationModel().get_email_tmpl(
                 reg_type, 'txt',
                 user=user.short_contact,
                 reset_token=token,
                 reset_url=link)
             html_body = notification.EmailNotificationModel().get_email_tmpl(
                 reg_type, 'html',
                 user=user.short_contact,
                 reset_token=token,
                 reset_url=link)
             log.debug('sending email')
             tasks.send_email([user_email], _("Password reset link"), body, html_body)
             log.info('send new password mail to %s', user_email)
         else:
             log.debug("password reset email %s not found", user_email)
 
         return webutils.url('reset_password_confirmation',
                      email=user_email,
                      timestamp=timestamp)
 
     def verify_reset_password_token(self, email, timestamp, token):
         user = db.User.get_by_email(email)
         if user is None:
             log.debug("user with email %s not found", email)
             return False
 
         token_age = int(time.time()) - int(timestamp)
 
         if token_age < 0:
             log.debug('timestamp is from the future')
             return False
 
         if token_age > UserModel.password_reset_token_lifetime:
             log.debug('password reset token expired')
             return False
 
         expected_token = self.get_reset_password_token(user,
                                                        timestamp,
                                                        webutils.session_csrf_secret_token())
         log.debug('computed password reset token: %s', expected_token)
         log.debug('received password reset token: %s', token)
         return expected_token == token
 
     def reset_password(self, user_email, new_passwd):
         from kallithea.lib import auth
         from kallithea.lib.celerylib import tasks
         user = db.User.get_by_email(user_email)
         if user is not None:
             if not self.can_change_password(user):
                 raise Exception('trying to change password for external user')
             user.password = auth.get_crypt_password(new_passwd)
             meta.Session().commit()
             log.info('change password for %s', user_email)
         if new_passwd is None:
             raise Exception('unable to set new password')
 
         tasks.send_email([user_email],
                  _('Password reset notification'),
                  _('The password to your account %s has been changed using password reset form.') % (user.username,))
         log.info('send password reset mail to %s', user_email)
 
         return True
 
     def has_perm(self, user, perm):
         perm = db.Permission.guess_instance(perm)
         user = db.User.guess_instance(user)
 
         return db.UserToPerm.query().filter(db.UserToPerm.user == user) \
             .filter(db.UserToPerm.permission == perm).scalar() is not None
 
     def grant_perm(self, user, perm):
         """
         Grant user global permissions
 
         :param user:
         :param perm:
         """
         user = db.User.guess_instance(user)
         perm = db.Permission.guess_instance(perm)
         # if this permission is already granted skip it
         _perm = db.UserToPerm.query() \
             .filter(db.UserToPerm.user == user) \
             .filter(db.UserToPerm.permission == perm) \
             .scalar()
         if _perm:
             return
         new = db.UserToPerm()
         new.user = user
         new.permission = perm
         meta.Session().add(new)
         return new
 
     def revoke_perm(self, user, perm):
         """
         Revoke users global permissions
 
         :param user:
         :param perm:
         """
         user = db.User.guess_instance(user)
         perm = db.Permission.guess_instance(perm)
 
         db.UserToPerm.query().filter(
             db.UserToPerm.user == user,
             db.UserToPerm.permission == perm,
         ).delete()
 
     def add_extra_email(self, user, email):
         """
         Adds email address to UserEmailMap
 
         :param user:
         :param email:
         """
-        from kallithea.model import forms
         form = forms.UserExtraEmailForm()()
         data = form.to_python(dict(email=email))
         user = db.User.guess_instance(user)
 
         obj = db.UserEmailMap()
         obj.user = user
         obj.email = data['email']
         meta.Session().add(obj)
         return obj
 
     def delete_extra_email(self, user, email_id):
         """
         Removes email address from UserEmailMap
 
         :param user:
         :param email_id:
         """
         user = db.User.guess_instance(user)
         obj = db.UserEmailMap.query().get(email_id)
         if obj is not None:
             meta.Session().delete(obj)
 
     def add_extra_ip(self, user, ip):
         """
         Adds IP address to UserIpMap
 
         :param user:
         :param ip:
         """
-        from kallithea.model import forms
         form = forms.UserExtraIpForm()()
         data = form.to_python(dict(ip=ip))
         user = db.User.guess_instance(user)
 
         obj = db.UserIpMap()
         obj.user = user
         obj.ip_addr = data['ip']
         meta.Session().add(obj)
         return obj
 
     def delete_extra_ip(self, user, ip_id):
         """
         Removes IP address from UserIpMap
 
         :param user:
         :param ip_id:
         """
         user = db.User.guess_instance(user)
         obj = db.UserIpMap.query().get(ip_id)
         if obj:
             meta.Session().delete(obj)

kallithea/model/validators.py

@@ -1,799 +1,799 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Set of generic validators
 """
 
 import logging
 import os
 import re
 from collections import defaultdict
 
 import formencode
 import ipaddr
 import sqlalchemy
 from formencode.validators import CIDR, Bool, Email, FancyValidator, Int, IPAddress, NotEmpty, Number, OneOf, Regex, Set, String, StringBoolean, UnicodeString
 from sqlalchemy import func
 from tg.i18n import ugettext as _
 
 import kallithea
-from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel
+from kallithea.lib import auth
 from kallithea.lib.compat import OrderedSet
 from kallithea.lib.exceptions import InvalidCloneUriException, LdapImportError
 from kallithea.lib.utils import is_valid_repo_uri
 from kallithea.lib.utils2 import asbool, aslist, repo_name_slug
 from kallithea.model import db
 
 
 # silence warnings and pylint
 UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set, \
     NotEmpty, IPAddress, CIDR, String, FancyValidator
 
 log = logging.getLogger(__name__)
 
 
 def UniqueListFromString():
     class _UniqueListFromString(formencode.FancyValidator):
         """
         Split value on ',' and make unique while preserving order
         """
         messages = dict(
             empty=_('Value cannot be an empty list'),
             missing_value=_('Value cannot be an empty list'),
         )
 
         def _convert_to_python(self, value, state):
             value = aslist(value, ',')
             seen = set()
             return [c for c in value if not (c in seen or seen.add(c))]
 
         def empty_value(self, value):
             return []
 
     return _UniqueListFromString
 
 
 def ValidUsername(edit=False, old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'username_exists': _('Username "%(username)s" already exists'),
             'system_invalid_username':
                 _('Username "%(username)s" cannot be used'),
             'invalid_username':
                 _('Username may only contain alphanumeric characters '
                   'underscores, periods or dashes and must begin with an '
                   'alphanumeric character or underscore')
         }
 
         def _validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 msg = self.message('system_invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state)
             # check if user is unique
             old_un = None
             if edit:
                 old_un = db.User.get(old_data.get('user_id')).username
 
             if old_un != value or not edit:
                 if db.User.get_by_username(value, case_insensitive=True):
                     msg = self.message('username_exists', state, username=value)
                     raise formencode.Invalid(msg, value, state)
 
             if re.match(r'^[a-zA-Z0-9\_]{1}[a-zA-Z0-9\-\_\.]*$', value) is None:
                 msg = self.message('invalid_username', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 
 
 def ValidRegex(msg=None):
     class _validator(formencode.validators.Regex):
         messages = dict(invalid=msg or _('The input is not valid'))
     return _validator
 
 
 def ValidRepoUser():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_username': _('Username %(username)s is not valid')
         }
 
         def _validate_python(self, value, state):
             try:
                 db.User.query().filter(db.User.active == True) \
                     .filter(db.User.username == value).one()
             except sqlalchemy.exc.InvalidRequestError: # NoResultFound/MultipleResultsFound
                 msg = self.message('invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(username=msg)
                 )
 
     return _validator
 
 
 def ValidUserGroup(edit=False, old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_group': _('Invalid user group name'),
             'group_exist': _('User group "%(usergroup)s" already exists'),
             'invalid_usergroup_name':
                 _('user group name may only contain alphanumeric '
                   'characters underscores, periods or dashes and must begin '
                   'with alphanumeric character')
         }
 
         def _validate_python(self, value, state):
             if value in ['default']:
                 msg = self.message('invalid_group', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
                 )
             # check if group is unique
             old_ugname = None
             if edit:
                 old_id = old_data.get('users_group_id')
                 old_ugname = db.UserGroup.get(old_id).users_group_name
 
             if old_ugname != value or not edit:
                 is_existing_group = db.UserGroup.get_by_group_name(value,
                                                         case_insensitive=True)
                 if is_existing_group:
                     msg = self.message('group_exist', state, usergroup=value)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(users_group_name=msg)
                     )
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 msg = self.message('invalid_usergroup_name', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
                 )
 
     return _validator
 
 
 def ValidRepoGroup(edit=False, old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'parent_group_id': _('Cannot assign this group as parent'),
             'group_exists': _('Group "%(group_name)s" already exists'),
             'repo_exists':
                 _('Repository with name "%(group_name)s" already exists')
         }
 
         def _validate_python(self, value, state):
             # TODO WRITE VALIDATIONS
             group_name = value.get('group_name')
             parent_group_id = value.get('parent_group_id')
 
             # slugify repo group just in case :)
             slug = repo_name_slug(group_name)
 
             # check for parent of self
             if edit and parent_group_id and old_data['group_id'] == parent_group_id:
                 msg = self.message('parent_group_id', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(parent_group_id=msg)
                 )
 
             old_gname = None
             if edit:
                 old_gname = db.RepoGroup.get(old_data.get('group_id')).group_name
 
             if old_gname != group_name or not edit:
 
                 # check group
                 gr = db.RepoGroup.query() \
                       .filter(func.lower(db.RepoGroup.group_name) == func.lower(slug)) \
                       .filter(db.RepoGroup.parent_group_id == parent_group_id) \
                       .scalar()
                 if gr is not None:
                     msg = self.message('group_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
                     )
 
                 # check for same repo
                 repo = db.Repository.query() \
                       .filter(func.lower(db.Repository.repo_name) == func.lower(slug)) \
                       .scalar()
                 if repo is not None:
                     msg = self.message('repo_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
                     )
 
     return _validator
 
 
 def ValidPassword():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password':
                 _('Invalid characters (non-ascii) in password')
         }
 
         def _validate_python(self, value, state):
             try:
                 (value or '').encode('ascii')
             except UnicodeError:
                 msg = self.message('invalid_password', state)
                 raise formencode.Invalid(msg, value, state,)
     return _validator
 
 
 def ValidOldPassword(username):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password': _('Invalid old password')
         }
 
         def _validate_python(self, value, state):
             from kallithea.lib import auth_modules
             if auth_modules.authenticate(username, value, '') is None:
                 msg = self.message('invalid_password', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(current_password=msg)
                 )
     return _validator
 
 
 def ValidPasswordsMatch(password_field, password_confirmation_field):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'password_mismatch': _('Passwords do not match'),
         }
 
         def _validate_python(self, value, state):
             if value.get(password_field) != value[password_confirmation_field]:
                 msg = self.message('password_mismatch', state)
                 raise formencode.Invalid(msg, value, state,
                      error_dict={password_field: msg, password_confirmation_field: msg}
                 )
     return _validator
 
 
 def ValidAuth():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_auth': _('Invalid username or password'),
         }
 
         def _validate_python(self, value, state):
             from kallithea.lib import auth_modules
 
             password = value['password']
             username = value['username']
 
             # authenticate returns unused dict but has called
             # plugin._authenticate which has create_or_update'ed the username user in db
             if auth_modules.authenticate(username, password) is None:
                 user = db.User.get_by_username_or_email(username)
                 if user and not user.active:
                     log.warning('user %s is disabled', username)
                     msg = self.message('invalid_auth', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=' ', password=msg)
                     )
                 else:
                     log.warning('user %s failed to authenticate', username)
                     msg = self.message('invalid_auth', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=' ', password=msg)
                     )
     return _validator
 
 
 def ValidRepoName(edit=False, old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_repo_name':
                 _('Repository name %(repo)s is not allowed'),
             'repository_exists':
                 _('Repository named %(repo)s already exists'),
             'repository_in_group_exists': _('Repository "%(repo)s" already '
                                             'exists in group "%(group)s"'),
             'same_group_exists': _('Repository group with name "%(repo)s" '
                                    'already exists')
         }
 
         def _convert_to_python(self, value, state):
             repo_name = repo_name_slug(value.get('repo_name', ''))
             repo_group = value.get('repo_group')
             if repo_group:
                 gr = db.RepoGroup.get(repo_group)
                 group_path = gr.full_path
                 group_name = gr.group_name
                 # value needs to be aware of group name in order to check
                 # db key This is an actual just the name to store in the
                 # database
                 repo_name_full = group_path + kallithea.URL_SEP + repo_name
             else:
                 group_name = group_path = ''
                 repo_name_full = repo_name
 
             value['repo_name'] = repo_name
             value['repo_name_full'] = repo_name_full
             value['group_path'] = group_path
             value['group_name'] = group_name
             return value
 
         def _validate_python(self, value, state):
             repo_name = value.get('repo_name')
             repo_name_full = value.get('repo_name_full')
             group_path = value.get('group_path')
             group_name = value.get('group_name')
 
             if repo_name in [kallithea.ADMIN_PREFIX, '']:
                 msg = self.message('invalid_repo_name', state, repo=repo_name)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_name=msg)
                 )
 
             rename = old_data.get('repo_name') != repo_name_full
             create = not edit
             if rename or create:
                 repo = db.Repository.get_by_repo_name(repo_name_full, case_insensitive=True)
                 repo_group = db.RepoGroup.get_by_group_name(repo_name_full, case_insensitive=True)
                 if group_path != '':
                     if repo is not None:
                         msg = self.message('repository_in_group_exists', state,
                                 repo=repo.repo_name, group=group_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
                         )
                 elif repo_group is not None:
                     msg = self.message('same_group_exists', state,
                             repo=repo_name)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_name=msg)
                     )
                 elif repo is not None:
                     msg = self.message('repository_exists', state,
                             repo=repo.repo_name)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_name=msg)
                     )
             return value
     return _validator
 
 
 def ValidForkName(*args, **kwargs):
     return ValidRepoName(*args, **kwargs)
 
 
 def SlugifyName():
     class _validator(formencode.validators.FancyValidator):
 
         def _convert_to_python(self, value, state):
             return repo_name_slug(value)
 
         def _validate_python(self, value, state):
             pass
 
     return _validator
 
 
 def ValidCloneUri():
     from kallithea.lib.utils import make_ui
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'clone_uri': _('Invalid repository URL'),
             'invalid_clone_uri': _('Invalid repository URL. It must be a '
                                    'valid http, https, or ssh URL'),
         }
 
         def _validate_python(self, value, state):
             repo_type = value.get('repo_type')
             url = value.get('clone_uri')
 
             if url and url != value.get('clone_uri_hidden'):
                 try:
                     is_valid_repo_uri(repo_type, url, make_ui())
                 except InvalidCloneUriException as e:
                     log.warning('validation of clone URL %r failed: %s', url, e)
                     msg = self.message('clone_uri', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(clone_uri=msg)
                     )
     return _validator
 
 
 def ValidForkType(old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_fork_type': _('Fork has to be the same type as parent')
         }
 
         def _validate_python(self, value, state):
             if old_data['repo_type'] != value:
                 msg = self.message('invalid_fork_type', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_type=msg)
                 )
     return _validator
 
 
 def CanWriteGroup(old_data=None):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _("You don't have permissions "
                                    "to create repository in this group"),
             'permission_denied_root': _("no permission to create repository "
                                         "in root location")
         }
 
         def _convert_to_python(self, value, state):
             # root location
             if value == -1:
                 return None
             return value
 
         def _validate_python(self, value, state):
             gr = db.RepoGroup.get(value)
             gr_name = gr.group_name if gr is not None else None # None means ROOT location
 
             # create repositories with write permission on group is set to true
-            group_admin = HasRepoGroupPermissionLevel('admin')(gr_name,
+            group_admin = auth.HasRepoGroupPermissionLevel('admin')(gr_name,
                                             'can write into group validator')
-            group_write = HasRepoGroupPermissionLevel('write')(gr_name,
+            group_write = auth.HasRepoGroupPermissionLevel('write')(gr_name,
                                             'can write into group validator')
             forbidden = not (group_admin or group_write)
-            can_create_repos = HasPermissionAny('hg.admin', 'hg.create.repository')
+            can_create_repos = auth.HasPermissionAny('hg.admin', 'hg.create.repository')
             gid = (old_data['repo_group'].get('group_id')
                    if (old_data and 'repo_group' in old_data) else None)
             value_changed = gid != value
             new = not old_data
             # do check if we changed the value, there's a case that someone got
             # revoked write permissions to a repository, he still created, we
             # don't need to check permission if he didn't change the value of
             # groups in form box
             if value_changed or new:
                 # parent group need to be existing
                 if gr and forbidden:
                     msg = self.message('permission_denied', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
                     )
                 ## check if we can write to root location !
                 elif gr is None and not can_create_repos():
                     msg = self.message('permission_denied_root', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
                     )
 
     return _validator
 
 
 def CanCreateGroup(can_create_in_root=False):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _("You don't have permissions "
                                    "to create a group in this location")
         }
 
         def to_python(self, value, state):
             # root location
             if value == -1:
                 return None
             return value
 
         def _validate_python(self, value, state):
             gr = db.RepoGroup.get(value)
             gr_name = gr.group_name if gr is not None else None # None means ROOT location
 
             if can_create_in_root and gr is None:
                 # we can create in root, we're fine no validations required
                 return
 
             forbidden_in_root = gr is None and not can_create_in_root
-            forbidden = not HasRepoGroupPermissionLevel('admin')(gr_name, 'can create group validator')
+            forbidden = not auth.HasRepoGroupPermissionLevel('admin')(gr_name, 'can create group validator')
             if forbidden_in_root or forbidden:
                 msg = self.message('permission_denied', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(parent_group_id=msg)
                 )
 
     return _validator
 
 
 def ValidPerms(type_='repo'):
     if type_ == 'repo_group':
         EMPTY_PERM = 'group.none'
     elif type_ == 'repo':
         EMPTY_PERM = 'repository.none'
     elif type_ == 'user_group':
         EMPTY_PERM = 'usergroup.none'
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'perm_new_member_name':
                 _('This username or user group name is not valid')
         }
 
         def to_python(self, value, state):
             perms_update = OrderedSet()
             perms_new = OrderedSet()
             # build a list of permission to update and new permission to create
 
             # CLEAN OUT ORG VALUE FROM NEW MEMBERS, and group them using
             new_perms_group = defaultdict(dict)
             for k, v in value.copy().items():
                 if k.startswith('perm_new_member'):
                     del value[k]
                     _type, part = k.split('perm_new_member_')
                     args = part.split('_')
                     if len(args) == 1:
                         new_perms_group[args[0]]['perm'] = v
                     elif len(args) == 2:
                         _key, pos = args
                         new_perms_group[pos][_key] = v
 
             # fill new permissions in order of how they were added
             for k in sorted(new_perms_group, key=lambda k: int(k)):
                 perm_dict = new_perms_group[k]
                 new_member = perm_dict.get('name')
                 new_perm = perm_dict.get('perm')
                 new_type = perm_dict.get('type')
                 if new_member and new_perm and new_type:
                     perms_new.add((new_member, new_perm, new_type))
 
             for k, v in value.items():
                 if k.startswith('u_perm_') or k.startswith('g_perm_'):
                     member_name = k[7:]
                     t = {'u': 'user',
                          'g': 'users_group'
                     }[k[0]]
                     if member_name == db.User.DEFAULT_USER_NAME:
                         if asbool(value.get('repo_private')):
                             # set none for default when updating to
                             # private repo protects against form manipulation
                             v = EMPTY_PERM
                     perms_update.add((member_name, v, t))
 
             value['perms_updates'] = list(perms_update)
             value['perms_new'] = list(perms_new)
 
             # update permissions
             for k, v, t in perms_new:
                 try:
                     if t == 'user':
                         _user_db = db.User.query() \
                             .filter(db.User.active == True) \
                             .filter(db.User.username == k).one()
                     if t == 'users_group':
                         _user_db = db.UserGroup.query() \
                             .filter(db.UserGroup.users_group_active == True) \
                             .filter(db.UserGroup.users_group_name == k).one()
 
                 except Exception as e:
                     log.warning('Error validating %s permission %s', t, k)
                     msg = self.message('perm_new_member_type', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(perm_new_member_name=msg)
                     )
             return value
     return _validator
 
 
 def ValidSettings():
     class _validator(formencode.validators.FancyValidator):
         def _convert_to_python(self, value, state):
             # settings  form for users that are not admin
             # can't edit certain parameters, it's extra backup if they mangle
             # with forms
 
             forbidden_params = [
                 'user', 'repo_type',
                 'repo_enable_downloads', 'repo_enable_statistics'
             ]
 
             for param in forbidden_params:
                 if param in value:
                     del value[param]
             return value
 
         def _validate_python(self, value, state):
             pass
     return _validator
 
 
 def ValidPath():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_path': _('This is not a valid path')
         }
 
         def _validate_python(self, value, state):
             if not os.path.isdir(value):
                 msg = self.message('invalid_path', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(paths_root_path=msg)
                 )
     return _validator
 
 
 def UniqSystemEmail(old_data=None):
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'email_taken': _('This email address is already in use')
         }
 
         def _convert_to_python(self, value, state):
             return value.lower()
 
         def _validate_python(self, value, state):
             if (old_data.get('email') or '').lower() != value:
                 user = db.User.get_by_email(value)
                 if user is not None:
                     msg = self.message('email_taken', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(email=msg)
                     )
     return _validator
 
 
 def ValidSystemEmail():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'non_existing_email': _('Email address "%(email)s" not found')
         }
 
         def _convert_to_python(self, value, state):
             return value.lower()
 
         def _validate_python(self, value, state):
             user = db.User.get_by_email(value)
             if user is None:
                 msg = self.message('non_existing_email', state, email=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(email=msg)
                 )
 
     return _validator
 
 
 def LdapLibValidator():
     class _validator(formencode.validators.FancyValidator):
         messages = {
 
         }
 
         def _validate_python(self, value, state):
             try:
                 import ldap
                 ldap  # pyflakes silence !
             except ImportError:
                 raise LdapImportError()
 
     return _validator
 
 
 def AttrLoginValidator():
     class _validator(formencode.validators.UnicodeString):
         messages = {
             'invalid_cn':
                   _('The LDAP Login attribute of the CN must be specified - '
                     'this is the name of the attribute that is equivalent '
                     'to "username"')
         }
         messages['empty'] = messages['invalid_cn']
 
     return _validator
 
 
 def ValidIp():
     class _validator(CIDR):
         messages = dict(
             badFormat=_('Please enter a valid IPv4 or IPv6 address'),
             illegalBits=_('The network size (bits) must be within the range'
                 ' of 0-32 (not %(bits)r)')
         )
 
         def to_python(self, value, state):
             v = super(_validator, self).to_python(value, state)
             v = v.strip()
             net = ipaddr.IPNetwork(address=v)
             if isinstance(net, ipaddr.IPv4Network):
                 # if IPv4 doesn't end with a mask, add /32
                 if '/' not in value:
                     v += '/32'
             if isinstance(net, ipaddr.IPv6Network):
                 # if IPv6 doesn't end with a mask, add /128
                 if '/' not in value:
                     v += '/128'
             return v
 
         def _validate_python(self, value, state):
             try:
                 addr = value.strip()
                 # this raises an ValueError if address is not IPv4 or IPv6
                 ipaddr.IPNetwork(address=addr)
             except ValueError:
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
 
     return _validator
 
 
 def FieldKey():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             badFormat=_('Key name can only consist of letters, '
                         'underscore, dash or numbers')
         )
 
         def _validate_python(self, value, state):
             if not re.match('[a-zA-Z0-9_-]+$', value):
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator
 
 
 def BasePath():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             badPath=_('Filename cannot be inside a directory')
         )
 
         def _convert_to_python(self, value, state):
             return value
 
         def _validate_python(self, value, state):
             if value != os.path.basename(value):
                 raise formencode.Invalid(self.message('badPath', state),
                                          value, state)
     return _validator
 
 
 def ValidAuthPlugins():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             import_duplicate=_('Plugins %(loaded)s and %(next_to_load)s both export the same name')
         )
 
         def _convert_to_python(self, value, state):
             # filter empty values
             return [s for s in value if s not in [None, '']]
 
         def _validate_python(self, value, state):
             from kallithea.lib import auth_modules
             module_list = value
             unique_names = {}
             try:
                 for module in module_list:
                     plugin = auth_modules.loadplugin(module)
                     plugin_name = plugin.name
                     if plugin_name in unique_names:
                         msg = self.message('import_duplicate', state,
                                 loaded=unique_names[plugin_name],
                                 next_to_load=plugin_name)
                         raise formencode.Invalid(msg, value, state)
                     unique_names[plugin_name] = plugin
             except (ImportError, AttributeError, TypeError) as e:
                 raise formencode.Invalid(str(e), value, state)
 
     return _validator

kallithea/tests/api/api_base.py

@@ -1,2821 +1,2821 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 Tests for the JSON-RPC web api.
 """
 
 import os
 import random
 import re
+import string
 
 import mock
 import pytest
 
 from kallithea.lib import ext_json
 from kallithea.lib.auth import AuthUser
 from kallithea.lib.utils2 import ascii_bytes
 from kallithea.model import db, meta
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.gist import GistModel
 from kallithea.model.pull_request import PullRequestModel
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.scm import ScmModel
 from kallithea.model.user import UserModel
 from kallithea.model.user_group import UserGroupModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture, raise_exception
 
 
 API_URL = '/_admin/api'
 TEST_USER_GROUP = 'test_user_group'
 TEST_REPO_GROUP = 'test_repo_group'
 
 fixture = Fixture()
 
 
 def _build_data(apikey, method, **kw):
     """
     Builds API data with given random ID
     For convenience, the json is returned as str
     """
     random_id = random.randrange(1, 9999)
     return random_id, ext_json.dumps({
         "id": random_id,
         "api_key": apikey,
         "method": method,
         "args": kw
     })
 
 
 jsonify = lambda obj: ext_json.loads(ext_json.dumps(obj))
 
 
 def api_call(test_obj, params):
     response = test_obj.app.post(API_URL, content_type='application/json',
                                  params=params)
     return response
 
 
 ## helpers
 def make_user_group(name=TEST_USER_GROUP):
     gr = fixture.create_user_group(name, cur_user=base.TEST_USER_ADMIN_LOGIN)
     UserGroupModel().add_user_to_group(user_group=gr,
                                        user=base.TEST_USER_ADMIN_LOGIN)
     meta.Session().commit()
     return gr
 
 
 def make_repo_group(name=TEST_REPO_GROUP):
     gr = fixture.create_repo_group(name, cur_user=base.TEST_USER_ADMIN_LOGIN)
     meta.Session().commit()
     return gr
 
 
 class _BaseTestApi(object):
     REPO = None
     REPO_TYPE = None
 
     @classmethod
     def setup_class(cls):
         cls.usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         cls.apikey = cls.usr.api_key
         cls.test_user = UserModel().create_or_update(
             username='test-api',
             password='test',
             email='[e-mail]',
             firstname='first',
             lastname='last'
         )
         meta.Session().commit()
         cls.TEST_USER_LOGIN = cls.test_user.username
         cls.apikey_regular = cls.test_user.api_key
 
     @classmethod
     def teardown_class(cls):
         pass
 
     def setup_method(self, method):
         make_user_group()
         make_repo_group()
 
     def teardown_method(self, method):
         fixture.destroy_user_group(TEST_USER_GROUP)
         fixture.destroy_gists()
         fixture.destroy_repo_group(TEST_REPO_GROUP)
 
     def _compare_ok(self, id_, expected, given):
         expected = jsonify({
             'id': id_,
             'error': None,
             'result': expected
         })
         given = ext_json.loads(given)
         assert expected == given, (expected, given)
 
     def _compare_error(self, id_, expected, given):
         expected = jsonify({
             'id': id_,
             'error': expected,
             'result': None
         })
         given = ext_json.loads(given)
         assert expected == given, (expected, given)
 
     def test_api_wrong_key(self):
         id_, params = _build_data('trololo', 'get_user')
         response = api_call(self, params)
 
         expected = 'Invalid API key'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_missing_non_optional_param(self):
         id_, params = _build_data(self.apikey, 'get_repo')
         response = api_call(self, params)
 
         expected = 'Missing non optional `repoid` arg in JSON DATA'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_missing_non_optional_param_args_null(self):
         id_, params = _build_data(self.apikey, 'get_repo')
         params = params.replace('"args": {}', '"args": null')
         response = api_call(self, params)
 
         expected = 'Missing non optional `repoid` arg in JSON DATA'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_missing_non_optional_param_args_bad(self):
         id_, params = _build_data(self.apikey, 'get_repo')
         params = params.replace('"args": {}', '"args": 1')
         response = api_call(self, params)
 
         expected = 'Missing non optional `repoid` arg in JSON DATA'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_args_is_null(self):
         id_, params = _build_data(self.apikey, 'get_users', )
         params = params.replace('"args": {}', '"args": null')
         response = api_call(self, params)
         assert response.status == '200 OK'
 
     def test_api_args_is_bad(self):
         id_, params = _build_data(self.apikey, 'get_users', )
         params = params.replace('"args": {}', '"args": 1')
         response = api_call(self, params)
         assert response.status == '200 OK'
 
     def test_api_args_different_args(self):
-        import string
         expected = {
             'ascii_letters': string.ascii_letters,
             'ws': string.whitespace,
             'printables': string.printable
         }
         id_, params = _build_data(self.apikey, 'test', args=expected)
         response = api_call(self, params)
         assert response.status == '200 OK'
         self._compare_ok(id_, expected, response.body)
 
     def test_api_get_users(self):
         id_, params = _build_data(self.apikey, 'get_users', )
         response = api_call(self, params)
         ret_all = []
         _users = db.User.query().filter_by(is_default_user=False) \
             .order_by(db.User.username).all()
         for usr in _users:
             ret = usr.get_api_data()
             ret_all.append(jsonify(ret))
         expected = ret_all
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_user(self):
         id_, params = _build_data(self.apikey, 'get_user',
                                   userid=base.TEST_USER_ADMIN_LOGIN)
         response = api_call(self, params)
 
         usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         ret = usr.get_api_data()
         ret['permissions'] = AuthUser(dbuser=usr).permissions
 
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_user_that_does_not_exist(self):
         id_, params = _build_data(self.apikey, 'get_user',
                                   userid='trololo')
         response = api_call(self, params)
 
         expected = "user `%s` does not exist" % 'trololo'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_user_without_giving_userid(self):
         id_, params = _build_data(self.apikey, 'get_user')
         response = api_call(self, params)
 
         usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         ret = usr.get_api_data()
         ret['permissions'] = AuthUser(dbuser=usr).permissions
 
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_user_without_giving_userid_non_admin(self):
         id_, params = _build_data(self.apikey_regular, 'get_user')
         response = api_call(self, params)
 
         usr = db.User.get_by_username(self.TEST_USER_LOGIN)
         ret = usr.get_api_data()
         ret['permissions'] = AuthUser(dbuser=usr).permissions
 
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_user_with_giving_userid_non_admin(self):
         id_, params = _build_data(self.apikey_regular, 'get_user',
                                   userid=self.TEST_USER_LOGIN)
         response = api_call(self, params)
 
         expected = 'userid is not the same as your user'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_pull_remote(self):
         # Note: pulling from local repos is a mis-feature - it will bypass access control
         # ... but ok, if the path already has been set in the database
         repo_name = 'test_pull'
         r = fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         # hack around that clone_uri can't be set to to a local path
         # (as shown by test_api_create_repo_clone_uri_local)
         r.clone_uri = os.path.join(db.Ui.get_by_key('paths', '/').ui_value, self.REPO)
         meta.Session().commit()
 
         pre_cached_tip = [repo.get_api_data()['last_changeset']['short_id'] for repo in db.Repository.query().filter(db.Repository.repo_name == repo_name)]
 
         id_, params = _build_data(self.apikey, 'pull',
                                   repoid=repo_name,)
         response = api_call(self, params)
 
         expected = {'msg': 'Pulled from `%s`' % repo_name,
                     'repository': repo_name}
         self._compare_ok(id_, expected, given=response.body)
 
         post_cached_tip = [repo.get_api_data()['last_changeset']['short_id'] for repo in db.Repository.query().filter(db.Repository.repo_name == repo_name)]
 
         fixture.destroy_repo(repo_name)
 
         assert pre_cached_tip != post_cached_tip
 
     def test_api_pull_fork(self):
         fork_name = 'fork'
         fixture.create_fork(self.REPO, fork_name)
         id_, params = _build_data(self.apikey, 'pull',
                                   repoid=fork_name,)
         response = api_call(self, params)
 
         expected = {'msg': 'Pulled from `%s`' % fork_name,
                     'repository': fork_name}
         self._compare_ok(id_, expected, given=response.body)
 
         fixture.destroy_repo(fork_name)
 
     def test_api_pull_error_no_remote_no_fork(self):
         # should fail because no clone_uri is set
         id_, params = _build_data(self.apikey, 'pull',
                                   repoid=self.REPO, )
         response = api_call(self, params)
 
         expected = 'Unable to pull changes from `%s`' % self.REPO
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_pull_custom_remote(self):
         repo_name = 'test_pull_custom_remote'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
 
         custom_remote_path = os.path.join(db.Ui.get_by_key('paths', '/').ui_value, self.REPO)
 
         id_, params = _build_data(self.apikey, 'pull',
                                   repoid=repo_name,
                                   clone_uri=custom_remote_path)
         response = api_call(self, params)
 
         expected = {'msg': 'Pulled from `%s`' % repo_name,
                     'repository': repo_name}
         self._compare_ok(id_, expected, given=response.body)
 
         fixture.destroy_repo(repo_name)
 
     def test_api_rescan_repos(self):
         id_, params = _build_data(self.apikey, 'rescan_repos')
         response = api_call(self, params)
 
         expected = {'added': [], 'removed': []}
         self._compare_ok(id_, expected, given=response.body)
 
     @mock.patch.object(ScmModel, 'repo_scan', raise_exception)
     def test_api_rescann_error(self):
         id_, params = _build_data(self.apikey, 'rescan_repos', )
         response = api_call(self, params)
 
         expected = 'Error occurred during rescan repositories action'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_existing_user(self):
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=base.TEST_USER_ADMIN_LOGIN,
                                   email='[e-mail]',
                                   password='trololo')
         response = api_call(self, params)
 
         expected = "user `%s` already exist" % base.TEST_USER_ADMIN_LOGIN
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_user_with_existing_email(self):
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=base.TEST_USER_ADMIN_LOGIN + 'new',
                                   email=base.TEST_USER_REGULAR_EMAIL,
                                   password='trololo')
         response = api_call(self, params)
 
         expected = "email `%s` already exist" % base.TEST_USER_REGULAR_EMAIL
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_user(self):
         username = 'test_new_api_user'
         email = username + "@example.com"
 
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=username,
                                   email=email,
                                   password='trololo')
         response = api_call(self, params)
 
         usr = db.User.get_by_username(username)
         ret = dict(
             msg='created new user `%s`' % username,
             user=jsonify(usr.get_api_data())
         )
 
         try:
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user(usr.user_id)
 
     def test_api_create_user_without_password(self):
         username = 'test_new_api_user_passwordless'
         email = username + "@example.com"
 
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=username,
                                   email=email)
         response = api_call(self, params)
 
         usr = db.User.get_by_username(username)
         ret = dict(
             msg='created new user `%s`' % username,
             user=jsonify(usr.get_api_data())
         )
         try:
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user(usr.user_id)
 
     def test_api_create_user_with_extern_name(self):
         username = 'test_new_api_user_passwordless'
         email = username + "@example.com"
 
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=username,
                                   email=email, extern_name='internal')
         response = api_call(self, params)
 
         usr = db.User.get_by_username(username)
         ret = dict(
             msg='created new user `%s`' % username,
             user=jsonify(usr.get_api_data())
         )
         try:
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user(usr.user_id)
 
     @mock.patch.object(UserModel, 'create_or_update', raise_exception)
     def test_api_create_user_when_exception_happened(self):
 
         username = 'test_new_api_user'
         email = username + "@example.com"
 
         id_, params = _build_data(self.apikey, 'create_user',
                                   username=username,
                                   email=email,
                                   password='trololo')
         response = api_call(self, params)
         expected = 'failed to create user `%s`' % username
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_delete_user(self):
         usr = UserModel().create_or_update(username='test_user',
                                            password='qweqwe',
                                            email='[e-mail]',
                                            firstname='u1', lastname='u1')
         meta.Session().commit()
         username = usr.username
         email = usr.email
         usr_id = usr.user_id
         ## DELETE THIS USER NOW
 
         id_, params = _build_data(self.apikey, 'delete_user',
                                   userid=username, )
         response = api_call(self, params)
 
         ret = {'msg': 'deleted user ID:%s %s' % (usr_id, username),
                'user': None}
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     @mock.patch.object(UserModel, 'delete', raise_exception)
     def test_api_delete_user_when_exception_happened(self):
         usr = UserModel().create_or_update(username='test_user',
                                            password='qweqwe',
                                            email='[e-mail]',
                                            firstname='u1', lastname='u1')
         meta.Session().commit()
         username = usr.username
 
         id_, params = _build_data(self.apikey, 'delete_user',
                                   userid=username, )
         response = api_call(self, params)
         ret = 'failed to delete user ID:%s %s' % (usr.user_id,
                                                   usr.username)
         expected = ret
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,expected', [
         ('firstname', 'new_username'),
         ('lastname', 'new_username'),
         ('email', 'new_username'),
         ('admin', True),
         ('admin', False),
         ('extern_type', 'ldap'),
         ('extern_type', None),
         ('extern_name', 'test'),
         ('extern_name', None),
         ('active', False),
         ('active', True),
         ('password', 'newpass'),
     ])
     def test_api_update_user(self, name, expected):
         usr = db.User.get_by_username(self.TEST_USER_LOGIN)
         kw = {name: expected,
               'userid': usr.user_id}
         id_, params = _build_data(self.apikey, 'update_user', **kw)
         response = api_call(self, params)
 
         ret = {
             'msg': 'updated user ID:%s %s' % (
                 usr.user_id, self.TEST_USER_LOGIN),
             'user': jsonify(db.User \
                 .get_by_username(self.TEST_USER_LOGIN) \
                 .get_api_data())
         }
 
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_update_user_no_changed_params(self):
         usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         ret = jsonify(usr.get_api_data())
         id_, params = _build_data(self.apikey, 'update_user',
                                   userid=base.TEST_USER_ADMIN_LOGIN)
 
         response = api_call(self, params)
         ret = {
             'msg': 'updated user ID:%s %s' % (
                 usr.user_id, base.TEST_USER_ADMIN_LOGIN),
             'user': ret
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_update_user_by_user_id(self):
         usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         ret = jsonify(usr.get_api_data())
         id_, params = _build_data(self.apikey, 'update_user',
                                   userid=usr.user_id)
 
         response = api_call(self, params)
         ret = {
             'msg': 'updated user ID:%s %s' % (
                 usr.user_id, base.TEST_USER_ADMIN_LOGIN),
             'user': ret
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_update_user_default_user(self):
         usr = db.User.get_default_user()
         id_, params = _build_data(self.apikey, 'update_user',
                                   userid=usr.user_id)
 
         response = api_call(self, params)
         expected = 'editing default user is forbidden'
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(UserModel, 'update_user', raise_exception)
     def test_api_update_user_when_exception_happens(self):
         usr = db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         ret = jsonify(usr.get_api_data())
         id_, params = _build_data(self.apikey, 'update_user',
                                   userid=usr.user_id)
 
         response = api_call(self, params)
         ret = 'failed to update user `%s`' % usr.user_id
 
         expected = ret
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_repo(self):
         new_group = 'some_new_group'
         make_user_group(new_group)
         RepoModel().grant_user_group_permission(repo=self.REPO,
                                                 group_name=new_group,
                                                 perm='repository.read')
         meta.Session().commit()
         id_, params = _build_data(self.apikey, 'get_repo',
                                   repoid=self.REPO)
         response = api_call(self, params)
         assert "tags" not in response.json['result']
         assert 'pull_requests' not in response.json['result']
 
         repo = RepoModel().get_by_repo_name(self.REPO)
         ret = repo.get_api_data()
 
         members = []
         followers = []
         for user in repo.repo_to_perm:
             perm = user.permission.permission_name
             user = user.user
             user_data = {'name': user.username, 'type': "user",
                          'permission': perm}
             members.append(user_data)
 
         for user_group in repo.users_group_to_perm:
             perm = user_group.permission.permission_name
             user_group = user_group.users_group
             user_group_data = {'name': user_group.users_group_name,
                                'type': "user_group", 'permission': perm}
             members.append(user_group_data)
 
         for user in repo.followers:
             followers.append(user.user.get_api_data())
 
         ret['members'] = members
         ret['followers'] = followers
 
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_user_group(new_group)
 
         id_, params = _build_data(self.apikey, 'get_repo', repoid=self.REPO,
                                   with_revision_names=True,
                                   with_pullrequests=True)
         response = api_call(self, params)
         assert "v0.2.0" in response.json['result']['tags']
         assert 'pull_requests' in response.json['result']
 
     @base.parametrize('grant_perm', [
         ('repository.admin'),
         ('repository.write'),
         ('repository.read'),
     ])
     def test_api_get_repo_by_non_admin(self, grant_perm):
         RepoModel().grant_user_permission(repo=self.REPO,
                                           user=self.TEST_USER_LOGIN,
                                           perm=grant_perm)
         meta.Session().commit()
         id_, params = _build_data(self.apikey_regular, 'get_repo',
                                   repoid=self.REPO)
         response = api_call(self, params)
 
         repo = RepoModel().get_by_repo_name(self.REPO)
         assert len(repo.repo_to_perm) >= 2  # make sure we actually are testing something - probably the default 2 permissions, possibly more
 
         expected = repo.get_api_data()
 
         members = []
         for user in repo.repo_to_perm:
             perm = user.permission.permission_name
             user_obj = user.user
             user_data = {'name': user_obj.username, 'type': "user",
                          'permission': perm}
             members.append(user_data)
         for user_group in repo.users_group_to_perm:
             perm = user_group.permission.permission_name
             user_group_obj = user_group.users_group
             user_group_data = {'name': user_group_obj.users_group_name,
                                'type': "user_group", 'permission': perm}
             members.append(user_group_data)
         expected['members'] = members
 
         followers = []
 
         for user in repo.followers:
             followers.append(user.user.get_api_data())
 
         expected['followers'] = followers
 
         try:
             self._compare_ok(id_, expected, given=response.body)
         finally:
             RepoModel().revoke_user_permission(self.REPO, self.TEST_USER_LOGIN)
 
     def test_api_get_repo_by_non_admin_no_permission_to_repo(self):
         RepoModel().grant_user_permission(repo=self.REPO,
                                           user=db.User.DEFAULT_USER_NAME,
                                           perm='repository.none')
         try:
             RepoModel().grant_user_permission(repo=self.REPO,
                                               user=self.TEST_USER_LOGIN,
                                               perm='repository.none')
 
             id_, params = _build_data(self.apikey_regular, 'get_repo',
                                       repoid=self.REPO)
             response = api_call(self, params)
 
             expected = 'repository `%s` does not exist' % (self.REPO)
             self._compare_error(id_, expected, given=response.body)
         finally:
             RepoModel().grant_user_permission(repo=self.REPO,
                                               user=db.User.DEFAULT_USER_NAME,
                                               perm='repository.read')
 
     def test_api_get_repo_that_doesn_not_exist(self):
         id_, params = _build_data(self.apikey, 'get_repo',
                                   repoid='no-such-repo')
         response = api_call(self, params)
 
         ret = 'repository `%s` does not exist' % 'no-such-repo'
         expected = ret
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_repos(self):
         id_, params = _build_data(self.apikey, 'get_repos')
         response = api_call(self, params)
 
         expected = jsonify([
             repo.get_api_data()
             for repo in db.Repository.query()
         ])
 
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_repos_non_admin(self):
         id_, params = _build_data(self.apikey_regular, 'get_repos')
         response = api_call(self, params)
 
         expected = jsonify([
             repo.get_api_data()
             for repo in RepoModel().get_all_user_repos(self.TEST_USER_LOGIN)
         ])
 
         self._compare_ok(id_, expected, given=response.body)
 
     @base.parametrize('name,ret_type', [
         ('all', 'all'),
         ('dirs', 'dirs'),
         ('files', 'files'),
     ])
     def test_api_get_repo_nodes(self, name, ret_type):
         rev = 'tip'
         path = '/'
         id_, params = _build_data(self.apikey, 'get_repo_nodes',
                                   repoid=self.REPO, revision=rev,
                                   root_path=path,
                                   ret_type=ret_type)
         response = api_call(self, params)
 
         # we don't the actual return types here since it's tested somewhere
         # else
         expected = response.json['result']
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_repo_nodes_bad_revisions(self):
         rev = 'i-dont-exist'
         path = '/'
         id_, params = _build_data(self.apikey, 'get_repo_nodes',
                                   repoid=self.REPO, revision=rev,
                                   root_path=path, )
         response = api_call(self, params)
 
         expected = 'failed to get repo: `%s` nodes' % self.REPO
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_repo_nodes_bad_path(self):
         rev = 'tip'
         path = '/idontexits'
         id_, params = _build_data(self.apikey, 'get_repo_nodes',
                                   repoid=self.REPO, revision=rev,
                                   root_path=path, )
         response = api_call(self, params)
 
         expected = 'failed to get repo: `%s` nodes' % self.REPO
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_repo_nodes_bad_ret_type(self):
         rev = 'tip'
         path = '/'
         ret_type = 'error'
         id_, params = _build_data(self.apikey, 'get_repo_nodes',
                                   repoid=self.REPO, revision=rev,
                                   root_path=path,
                                   ret_type=ret_type)
         response = api_call(self, params)
 
         expected = ('ret_type must be one of %s'
                     % (','.join(sorted(['files', 'dirs', 'all']))))
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,ret_type,grant_perm', [
         ('all', 'all', 'repository.write'),
         ('dirs', 'dirs', 'repository.admin'),
         ('files', 'files', 'repository.read'),
     ])
     def test_api_get_repo_nodes_by_regular_user(self, name, ret_type, grant_perm):
         RepoModel().grant_user_permission(repo=self.REPO,
                                           user=self.TEST_USER_LOGIN,
                                           perm=grant_perm)
         meta.Session().commit()
 
         rev = 'tip'
         path = '/'
         id_, params = _build_data(self.apikey_regular, 'get_repo_nodes',
                                   repoid=self.REPO, revision=rev,
                                   root_path=path,
                                   ret_type=ret_type)
         response = api_call(self, params)
 
         # we don't the actual return types here since it's tested somewhere
         # else
         expected = response.json['result']
         try:
             self._compare_ok(id_, expected, given=response.body)
         finally:
             RepoModel().revoke_user_permission(self.REPO, self.TEST_USER_LOGIN)
 
     def test_api_create_repo(self):
         repo_name = 'api-repo'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     @base.parametrize('repo_name', [
         '',
         '.',
         '..',
         ':',
         '/',
         '<test>',
     ])
     def test_api_create_repo_bad_names(self, repo_name):
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
         if repo_name == '/':
             expected = "repo group `` not found"
             self._compare_error(id_, expected, given=response.body)
         else:
             expected = "failed to create repository `%s`" % repo_name
             self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_clone_uri_local(self):
         # cloning from local repos was a mis-feature - it would bypass access control
         # TODO: introduce other test coverage of actual remote cloning
         clone_uri = os.path.join(base.TESTS_TMP_PATH, self.REPO)
         repo_name = 'api-repo'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,
                                   clone_uri=clone_uri,
         )
         response = api_call(self, params)
         expected = "failed to create repository `%s`" % repo_name
         self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_and_repo_group(self):
         repo_group_name = 'my_gr'
         repo_name = '%s/api-repo' % repo_group_name
 
         # repo creation can no longer also create repo group
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = 'repo group `%s` not found' % repo_group_name
         self._compare_error(id_, expected, given=response.body)
         assert RepoModel().get_by_repo_name(repo_name) is None
 
         # create group before creating repo
         rg = fixture.create_repo_group(repo_group_name)
         meta.Session().commit()
 
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
         }
         self._compare_ok(id_, expected, given=response.body)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
 
         fixture.destroy_repo(repo_name)
         fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_in_repo_group_without_permission(self):
         repo_group_basename = 'api-repo-repo'
         repo_group_name = '%s/%s' % (TEST_REPO_GROUP, repo_group_basename)
         repo_name = '%s/api-repo' % repo_group_name
 
         top_group = db.RepoGroup.get_by_group_name(TEST_REPO_GROUP)
         assert top_group
         rg = fixture.create_repo_group(repo_group_basename, parent_group_id=top_group)
         meta.Session().commit()
         RepoGroupModel().grant_user_permission(repo_group_name,
                                                self.TEST_USER_LOGIN,
                                                'group.none')
         meta.Session().commit()
 
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
 
         # Current result when API access control is different from Web:
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
         # Expected and arguably more correct result:
         #expected = 'failed to create repository `%s`' % repo_name
         #self._compare_error(id_, expected, given=response.body)
 
         fixture.destroy_repo_group(repo_group_name)
 
     def test_api_create_repo_unknown_owner(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=owner,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
         expected = 'user `%s` does not exist' % owner
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_repo_dont_specify_owner(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_by_non_admin(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
         )
         response = api_call(self, params)
 
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_by_non_admin_specify_owner(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
                                   owner=owner)
         response = api_call(self, params)
 
         expected = 'Only Kallithea admin can specify `owner` param'
         self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     def test_api_create_repo_exists(self):
         repo_name = self.REPO
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = "repo `%s` already exist" % repo_name
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_repo_dot_dot(self):
         # it is only possible to create repositories in existing repo groups - and '..' can't be used
         group_name = '%s/..' % TEST_REPO_GROUP
         repo_name = '%s/%s' % (group_name, 'could-be-outside')
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = 'repo group `%s` not found' % group_name
         self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
 
     @mock.patch.object(RepoModel, 'create', raise_exception)
     def test_api_create_repo_exception_occurred(self):
         repo_name = 'api-repo'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = 'failed to create repository `%s`' % repo_name
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('changing_attr,updates', [
         ('owner', {'owner': base.TEST_USER_REGULAR_LOGIN}),
         ('description', {'description': 'new description'}),
         ('clone_uri', {'clone_uri': 'http://example.com/repo'}), # will fail - pulling from non-existing repo should fail
         ('clone_uri', {'clone_uri': '/repo'}), # will fail - pulling from local repo was a mis-feature - it would bypass access control
         ('clone_uri', {'clone_uri': None}),
         ('landing_rev', {'landing_rev': 'branch:master'}),
         ('enable_statistics', {'enable_statistics': True}),
         ('enable_downloads', {'enable_downloads': True}),
         ('name', {'name': 'new_repo_name'}),
         ('repo_group', {'group': 'test_group_for_update'}),
     ])
     def test_api_update_repo(self, changing_attr, updates):
         repo_name = 'api_update_me'
         repo = fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         if changing_attr == 'repo_group':
             fixture.create_repo_group(updates['group'])
 
         id_, params = _build_data(self.apikey, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         if changing_attr == 'name':
             repo_name = updates['name']
         if changing_attr == 'repo_group':
             repo_name = '/'.join([updates['group'], repo_name])
         try:
             if changing_attr == 'clone_uri' and updates['clone_uri']:
                 expected = 'failed to update repo `%s`' % repo_name
                 self._compare_error(id_, expected, given=response.body)
             else:
                 expected = {
                     'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo_name),
                     'repository': repo.get_api_data()
                 }
                 self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
             if changing_attr == 'repo_group':
                 fixture.destroy_repo_group(updates['group'])
 
     @base.parametrize('changing_attr,updates', [
         ('owner', {'owner': base.TEST_USER_REGULAR_LOGIN}),
         ('description', {'description': 'new description'}),
         ('clone_uri', {'clone_uri': 'http://example.com/repo'}), # will fail - pulling from non-existing repo should fail
         ('clone_uri', {'clone_uri': '/repo'}), # will fail - pulling from local repo was a mis-feature - it would bypass access control
         ('clone_uri', {'clone_uri': None}),
         ('landing_rev', {'landing_rev': 'branch:master'}),
         ('enable_statistics', {'enable_statistics': True}),
         ('enable_downloads', {'enable_downloads': True}),
         ('name', {'name': 'new_repo_name'}),
         ('repo_group', {'group': 'test_group_for_update'}),
     ])
     def test_api_update_group_repo(self, changing_attr, updates):
         group_name = 'lololo'
         fixture.create_repo_group(group_name)
         repo_name = '%s/api_update_me' % group_name
         repo = fixture.create_repo(repo_name, repo_group=group_name, repo_type=self.REPO_TYPE)
         if changing_attr == 'repo_group':
             fixture.create_repo_group(updates['group'])
 
         id_, params = _build_data(self.apikey, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         if changing_attr == 'name':
             repo_name = '%s/%s' % (group_name, updates['name'])
         if changing_attr == 'repo_group':
             repo_name = '/'.join([updates['group'], repo_name.rsplit('/', 1)[-1]])
         try:
             if changing_attr == 'clone_uri' and updates['clone_uri']:
                 expected = 'failed to update repo `%s`' % repo_name
                 self._compare_error(id_, expected, given=response.body)
             else:
                 expected = {
                     'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo_name),
                     'repository': repo.get_api_data()
                 }
                 self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
             if changing_attr == 'repo_group':
                 fixture.destroy_repo_group(updates['group'])
         fixture.destroy_repo_group(group_name)
 
     def test_api_update_repo_repo_group_does_not_exist(self):
         repo_name = 'admin_owned'
         fixture.create_repo(repo_name)
         updates = {'group': 'test_group_for_update'}
         id_, params = _build_data(self.apikey, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         try:
             expected = 'repository group `%s` does not exist' % updates['group']
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_update_repo_regular_user_not_allowed(self):
         repo_name = 'admin_owned'
         fixture.create_repo(repo_name)
         updates = {'description': 'something else'}
         id_, params = _build_data(self.apikey_regular, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         try:
             expected = 'repository `%s` does not exist' % repo_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     @mock.patch.object(RepoModel, 'update', raise_exception)
     def test_api_update_repo_exception_occurred(self):
         repo_name = 'api_update_me'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         id_, params = _build_data(self.apikey, 'update_repo',
                                   repoid=repo_name, owner=base.TEST_USER_ADMIN_LOGIN,)
         response = api_call(self, params)
         try:
             expected = 'failed to update repo `%s`' % repo_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_update_repo_regular_user_change_repo_name(self):
         repo_name = 'admin_owned'
         new_repo_name = 'new_repo_name'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         RepoModel().grant_user_permission(repo=repo_name,
                                           user=self.TEST_USER_LOGIN,
                                           perm='repository.admin')
         UserModel().revoke_perm('default', 'hg.create.repository')
         UserModel().grant_perm('default', 'hg.create.none')
         updates = {'name': new_repo_name}
         id_, params = _build_data(self.apikey_regular, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         try:
             expected = 'no permission to create (or move) repositories'
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
             fixture.destroy_repo(new_repo_name)
 
     def test_api_update_repo_regular_user_change_repo_name_allowed(self):
         repo_name = 'admin_owned'
         new_repo_name = 'new_repo_name'
         repo = fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         RepoModel().grant_user_permission(repo=repo_name,
                                           user=self.TEST_USER_LOGIN,
                                           perm='repository.admin')
         UserModel().revoke_perm('default', 'hg.create.none')
         UserModel().grant_perm('default', 'hg.create.repository')
         updates = {'name': new_repo_name}
         id_, params = _build_data(self.apikey_regular, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         try:
             expected = {
                 'msg': 'updated repo ID:%s %s' % (repo.repo_id, new_repo_name),
                 'repository': repo.get_api_data()
             }
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
             fixture.destroy_repo(new_repo_name)
 
     def test_api_update_repo_regular_user_change_owner(self):
         repo_name = 'admin_owned'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         RepoModel().grant_user_permission(repo=repo_name,
                                           user=self.TEST_USER_LOGIN,
                                           perm='repository.admin')
         updates = {'owner': base.TEST_USER_ADMIN_LOGIN}
         id_, params = _build_data(self.apikey_regular, 'update_repo',
                                   repoid=repo_name, **updates)
         response = api_call(self, params)
         try:
             expected = 'Only Kallithea admin can specify `owner` param'
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_delete_repo(self):
         repo_name = 'api_delete_me'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
 
         id_, params = _build_data(self.apikey, 'delete_repo',
                                   repoid=repo_name, )
         response = api_call(self, params)
 
         ret = {
             'msg': 'Deleted repository `%s`' % repo_name,
             'success': True
         }
         try:
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_delete_repo_by_non_admin(self):
         repo_name = 'api_delete_me'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE,
                             cur_user=self.TEST_USER_LOGIN)
         id_, params = _build_data(self.apikey_regular, 'delete_repo',
                                   repoid=repo_name, )
         response = api_call(self, params)
 
         ret = {
             'msg': 'Deleted repository `%s`' % repo_name,
             'success': True
         }
         try:
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_delete_repo_by_non_admin_no_permission(self):
         repo_name = 'api_delete_me'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         try:
             id_, params = _build_data(self.apikey_regular, 'delete_repo',
                                       repoid=repo_name, )
             response = api_call(self, params)
             expected = 'repository `%s` does not exist' % (repo_name)
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_delete_repo_exception_occurred(self):
         repo_name = 'api_delete_me'
         fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)
         try:
             with mock.patch.object(RepoModel, 'delete', raise_exception):
                 id_, params = _build_data(self.apikey, 'delete_repo',
                                           repoid=repo_name, )
                 response = api_call(self, params)
 
                 expected = 'failed to delete repository `%s`' % repo_name
                 self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_fork_repo(self):
         fork_name = 'api-repo-fork'
         id_, params = _build_data(self.apikey, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
         )
         response = api_call(self, params)
 
         ret = {
             'msg': 'Created fork of `%s` as `%s`' % (self.REPO,
                                                      fork_name),
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
 
     @base.parametrize('fork_name', [
         'api-repo-fork',
         '%s/api-repo-fork' % TEST_REPO_GROUP,
     ])
     def test_api_fork_repo_non_admin(self, fork_name):
         id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
         )
         response = api_call(self, params)
 
         ret = {
             'msg': 'Created fork of `%s` as `%s`' % (self.REPO,
                                                      fork_name),
             'success': True,
             'task': None,
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
 
     def test_api_fork_repo_non_admin_specify_owner(self):
         fork_name = 'api-repo-fork'
         id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
         )
         response = api_call(self, params)
         expected = 'Only Kallithea admin can specify `owner` param'
         self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
 
     def test_api_fork_repo_non_admin_no_permission_to_fork(self):
         RepoModel().grant_user_permission(repo=self.REPO,
                                           user=db.User.DEFAULT_USER_NAME,
                                           perm='repository.none')
         try:
             fork_name = 'api-repo-fork'
             id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                       repoid=self.REPO,
                                       fork_name=fork_name,
             )
             response = api_call(self, params)
             expected = 'repository `%s` does not exist' % (self.REPO)
             self._compare_error(id_, expected, given=response.body)
         finally:
             RepoModel().grant_user_permission(repo=self.REPO,
                                               user=db.User.DEFAULT_USER_NAME,
                                               perm='repository.read')
             fixture.destroy_repo(fork_name)
 
     @base.parametrize('name,perm', [
         ('read', 'repository.read'),
         ('write', 'repository.write'),
         ('admin', 'repository.admin'),
     ])
     def test_api_fork_repo_non_admin_no_create_repo_permission(self, name, perm):
         fork_name = 'api-repo-fork'
         # regardless of base repository permission, forking is disallowed
         # when repository creation is disabled
         RepoModel().grant_user_permission(repo=self.REPO,
                                           user=self.TEST_USER_LOGIN,
                                           perm=perm)
         UserModel().revoke_perm('default', 'hg.create.repository')
         UserModel().grant_perm('default', 'hg.create.none')
         id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
         )
         response = api_call(self, params)
         expected = 'no permission to create repositories'
         self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
 
     def test_api_fork_repo_unknown_owner(self):
         fork_name = 'api-repo-fork'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=owner,
         )
         response = api_call(self, params)
         expected = 'user `%s` does not exist' % owner
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_fork_repo_fork_exists(self):
         fork_name = 'api-repo-fork'
         fixture.create_fork(self.REPO, fork_name)
 
         try:
             fork_name = 'api-repo-fork'
 
             id_, params = _build_data(self.apikey, 'fork_repo',
                                       repoid=self.REPO,
                                       fork_name=fork_name,
                                       owner=base.TEST_USER_ADMIN_LOGIN,
             )
             response = api_call(self, params)
 
             expected = "fork `%s` already exist" % fork_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(fork_name)
 
     def test_api_fork_repo_repo_exists(self):
         fork_name = self.REPO
 
         id_, params = _build_data(self.apikey, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
         )
         response = api_call(self, params)
 
         expected = "repo `%s` already exist" % fork_name
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'create_fork', raise_exception)
     def test_api_fork_repo_exception_occurred(self):
         fork_name = 'api-repo-fork'
         id_, params = _build_data(self.apikey, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
         )
         response = api_call(self, params)
 
         expected = 'failed to fork repository `%s` as `%s`' % (self.REPO,
                                                                fork_name)
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_user_group(self):
         id_, params = _build_data(self.apikey, 'get_user_group',
                                   usergroupid=TEST_USER_GROUP)
         response = api_call(self, params)
 
         user_group = UserGroupModel().get_group(TEST_USER_GROUP)
         members = []
         for user in user_group.members:
             user = user.user
             members.append(user.get_api_data())
 
         ret = user_group.get_api_data()
         ret['members'] = members
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_user_groups(self):
         gr_name = 'test_user_group2'
         make_user_group(gr_name)
 
         try:
             id_, params = _build_data(self.apikey, 'get_user_groups', )
             response = api_call(self, params)
 
             expected = []
             for gr_name in [TEST_USER_GROUP, 'test_user_group2']:
                 user_group = UserGroupModel().get_group(gr_name)
                 ret = user_group.get_api_data()
                 expected.append(ret)
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     def test_api_create_user_group(self):
         group_name = 'some_new_group'
         id_, params = _build_data(self.apikey, 'create_user_group',
                                   group_name=group_name)
         response = api_call(self, params)
 
         ret = {
             'msg': 'created new user group `%s`' % group_name,
             'user_group': jsonify(UserGroupModel() \
                 .get_by_name(group_name) \
                 .get_api_data())
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
         fixture.destroy_user_group(group_name)
 
     def test_api_get_user_group_that_exist(self):
         id_, params = _build_data(self.apikey, 'create_user_group',
                                   group_name=TEST_USER_GROUP)
         response = api_call(self, params)
 
         expected = "user group `%s` already exist" % TEST_USER_GROUP
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(UserGroupModel, 'create', raise_exception)
     def test_api_get_user_group_exception_occurred(self):
         group_name = 'exception_happens'
         id_, params = _build_data(self.apikey, 'create_user_group',
                                   group_name=group_name)
         response = api_call(self, params)
 
         expected = 'failed to create group `%s`' % group_name
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('changing_attr,updates', [
         ('group_name', {'group_name': 'new_group_name'}),
         ('group_name', {'group_name': 'test_group_for_update'}),
         ('owner', {'owner': base.TEST_USER_REGULAR_LOGIN}),
         ('active', {'active': False}),
         ('active', {'active': True}),
     ])
     def test_api_update_user_group(self, changing_attr, updates):
         gr_name = 'test_group_for_update'
         user_group = fixture.create_user_group(gr_name)
         try:
             id_, params = _build_data(self.apikey, 'update_user_group',
                                       usergroupid=gr_name, **updates)
             response = api_call(self, params)
             expected = {
                'msg': 'updated user group ID:%s %s' % (user_group.users_group_id,
                                                      user_group.users_group_name),
                'user_group': user_group.get_api_data()
             }
             self._compare_ok(id_, expected, given=response.body)
         finally:
             if changing_attr == 'group_name':
                 # switch to updated name for proper cleanup
                 gr_name = updates['group_name']
             fixture.destroy_user_group(gr_name)
 
     @mock.patch.object(UserGroupModel, 'update', raise_exception)
     def test_api_update_user_group_exception_occurred(self):
         gr_name = 'test_group'
         fixture.create_user_group(gr_name)
         try:
             id_, params = _build_data(self.apikey, 'update_user_group',
                                       usergroupid=gr_name)
             response = api_call(self, params)
             expected = 'failed to update user group `%s`' % gr_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     def test_api_add_user_to_user_group(self):
         gr_name = 'test_group'
         fixture.create_user_group(gr_name)
         try:
             id_, params = _build_data(self.apikey, 'add_user_to_user_group',
                                       usergroupid=gr_name,
                                       userid=base.TEST_USER_ADMIN_LOGIN)
             response = api_call(self, params)
             expected = {
             'msg': 'added member `%s` to user group `%s`' % (
                     base.TEST_USER_ADMIN_LOGIN, gr_name),
             'success': True
             }
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     def test_api_add_user_to_user_group_that_doesnt_exist(self):
         id_, params = _build_data(self.apikey, 'add_user_to_user_group',
                                   usergroupid='false-group',
                                   userid=base.TEST_USER_ADMIN_LOGIN)
         response = api_call(self, params)
 
         expected = 'user group `%s` does not exist' % 'false-group'
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(UserGroupModel, 'add_user_to_group', raise_exception)
     def test_api_add_user_to_user_group_exception_occurred(self):
         gr_name = 'test_group'
         fixture.create_user_group(gr_name)
         try:
             id_, params = _build_data(self.apikey, 'add_user_to_user_group',
                                       usergroupid=gr_name,
                                       userid=base.TEST_USER_ADMIN_LOGIN)
             response = api_call(self, params)
             expected = 'failed to add member to user group `%s`' % gr_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     def test_api_remove_user_from_user_group(self):
         gr_name = 'test_group_3'
         gr = fixture.create_user_group(gr_name)
         UserGroupModel().add_user_to_group(gr, user=base.TEST_USER_ADMIN_LOGIN)
         meta.Session().commit()
         try:
             id_, params = _build_data(self.apikey, 'remove_user_from_user_group',
                                       usergroupid=gr_name,
                                       userid=base.TEST_USER_ADMIN_LOGIN)
             response = api_call(self, params)
             expected = {
                 'msg': 'removed member `%s` from user group `%s`' % (
                     base.TEST_USER_ADMIN_LOGIN, gr_name
                 ),
                 'success': True}
             self._compare_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     @mock.patch.object(UserGroupModel, 'remove_user_from_group', raise_exception)
     def test_api_remove_user_from_user_group_exception_occurred(self):
         gr_name = 'test_group_3'
         gr = fixture.create_user_group(gr_name)
         UserGroupModel().add_user_to_group(gr, user=base.TEST_USER_ADMIN_LOGIN)
         meta.Session().commit()
         try:
             id_, params = _build_data(self.apikey, 'remove_user_from_user_group',
                                       usergroupid=gr_name,
                                       userid=base.TEST_USER_ADMIN_LOGIN)
             response = api_call(self, params)
             expected = 'failed to remove member from user group `%s`' % gr_name
             self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     def test_api_delete_user_group(self):
         gr_name = 'test_group'
         ugroup = fixture.create_user_group(gr_name)
         gr_id = ugroup.users_group_id
         try:
             id_, params = _build_data(self.apikey, 'delete_user_group',
                                       usergroupid=gr_name)
             response = api_call(self, params)
             expected = {
                 'user_group': None,
                 'msg': 'deleted user group ID:%s %s' % (gr_id, gr_name)
             }
             self._compare_ok(id_, expected, given=response.body)
         finally:
             if UserGroupModel().get_by_name(gr_name):
                 fixture.destroy_user_group(gr_name)
 
     def test_api_delete_user_group_that_is_assigned(self):
         gr_name = 'test_group'
         ugroup = fixture.create_user_group(gr_name)
         gr_id = ugroup.users_group_id
 
         ugr_to_perm = RepoModel().grant_user_group_permission(self.REPO, gr_name, 'repository.write')
         msg = 'User Group assigned to %s' % ugr_to_perm.repository.repo_name
 
         try:
             id_, params = _build_data(self.apikey, 'delete_user_group',
                                       usergroupid=gr_name)
             response = api_call(self, params)
             expected = msg
             self._compare_error(id_, expected, given=response.body)
         finally:
             if UserGroupModel().get_by_name(gr_name):
                 fixture.destroy_user_group(gr_name)
 
     def test_api_delete_user_group_exception_occurred(self):
         gr_name = 'test_group'
         ugroup = fixture.create_user_group(gr_name)
         gr_id = ugroup.users_group_id
         id_, params = _build_data(self.apikey, 'delete_user_group',
                                   usergroupid=gr_name)
 
         try:
             with mock.patch.object(UserGroupModel, 'delete', raise_exception):
                 response = api_call(self, params)
                 expected = 'failed to delete user group ID:%s %s' % (gr_id, gr_name)
                 self._compare_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_user_group(gr_name)
 
     @base.parametrize('name,perm', [
         ('none', 'repository.none'),
         ('read', 'repository.read'),
         ('write', 'repository.write'),
         ('admin', 'repository.admin'),
     ])
     def test_api_grant_user_permission(self, name, perm):
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission',
                                   repoid=self.REPO,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm)
         response = api_call(self, params)
 
         ret = {
             'msg': 'Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                 perm, base.TEST_USER_ADMIN_LOGIN, self.REPO
             ),
             'success': True
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_grant_user_permission_wrong_permission(self):
         perm = 'haha.no.permission'
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission',
                                   repoid=self.REPO,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'permission `%s` does not exist' % perm
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'grant_user_permission', raise_exception)
     def test_api_grant_user_permission_exception_when_adding(self):
         perm = 'repository.read'
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission',
                                   repoid=self.REPO,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user: `%s` in repo: `%s`' % (
             base.TEST_USER_ADMIN_LOGIN, self.REPO
         )
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_revoke_user_permission(self):
         id_, params = _build_data(self.apikey,
                                   'revoke_user_permission',
                                   repoid=self.REPO,
                                   userid=base.TEST_USER_ADMIN_LOGIN, )
         response = api_call(self, params)
 
         expected = {
             'msg': 'Revoked perm for user: `%s` in repo: `%s`' % (
                 base.TEST_USER_ADMIN_LOGIN, self.REPO
             ),
             'success': True
         }
         self._compare_ok(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'revoke_user_permission', raise_exception)
     def test_api_revoke_user_permission_exception_when_adding(self):
         id_, params = _build_data(self.apikey,
                                   'revoke_user_permission',
                                   repoid=self.REPO,
                                   userid=base.TEST_USER_ADMIN_LOGIN, )
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user: `%s` in repo: `%s`' % (
             base.TEST_USER_ADMIN_LOGIN, self.REPO
         )
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,perm', [
         ('none', 'repository.none'),
         ('read', 'repository.read'),
         ('write', 'repository.write'),
         ('admin', 'repository.admin'),
     ])
     def test_api_grant_user_group_permission(self, name, perm):
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission',
                                   repoid=self.REPO,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm)
         response = api_call(self, params)
 
         ret = {
             'msg': 'Granted perm: `%s` for user group: `%s` in repo: `%s`' % (
                 perm, TEST_USER_GROUP, self.REPO
             ),
             'success': True
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_grant_user_group_permission_wrong_permission(self):
         perm = 'haha.no.permission'
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission',
                                   repoid=self.REPO,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'permission `%s` does not exist' % perm
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'grant_user_group_permission', raise_exception)
     def test_api_grant_user_group_permission_exception_when_adding(self):
         perm = 'repository.read'
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission',
                                   repoid=self.REPO,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user group: `%s` in repo: `%s`' % (
             TEST_USER_GROUP, self.REPO
         )
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_revoke_user_group_permission(self):
         RepoModel().grant_user_group_permission(repo=self.REPO,
                                                 group_name=TEST_USER_GROUP,
                                                 perm='repository.read')
         meta.Session().commit()
         id_, params = _build_data(self.apikey,
                                   'revoke_user_group_permission',
                                   repoid=self.REPO,
                                   usergroupid=TEST_USER_GROUP, )
         response = api_call(self, params)
 
         expected = {
             'msg': 'Revoked perm for user group: `%s` in repo: `%s`' % (
                 TEST_USER_GROUP, self.REPO
             ),
             'success': True
         }
         self._compare_ok(id_, expected, given=response.body)
 
     @mock.patch.object(RepoModel, 'revoke_user_group_permission', raise_exception)
     def test_api_revoke_user_group_permission_exception_when_adding(self):
         id_, params = _build_data(self.apikey,
                                   'revoke_user_group_permission',
                                   repoid=self.REPO,
                                   usergroupid=TEST_USER_GROUP, )
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user group: `%s` in repo: `%s`' % (
             TEST_USER_GROUP, self.REPO
         )
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,perm,apply_to_children', [
         ('none', 'group.none', 'none'),
         ('read', 'group.read', 'none'),
         ('write', 'group.write', 'none'),
         ('admin', 'group.admin', 'none'),
 
         ('none', 'group.none', 'all'),
         ('read', 'group.read', 'all'),
         ('write', 'group.write', 'all'),
         ('admin', 'group.admin', 'all'),
 
         ('none', 'group.none', 'repos'),
         ('read', 'group.read', 'repos'),
         ('write', 'group.write', 'repos'),
         ('admin', 'group.admin', 'repos'),
 
         ('none', 'group.none', 'groups'),
         ('read', 'group.read', 'groups'),
         ('write', 'group.write', 'groups'),
         ('admin', 'group.admin', 'groups'),
     ])
     def test_api_grant_user_permission_to_repo_group(self, name, perm, apply_to_children):
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm, apply_to_children=apply_to_children)
         response = api_call(self, params)
 
         ret = {
             'msg': 'Granted perm: `%s` (recursive:%s) for user: `%s` in repo group: `%s`' % (
                 perm, apply_to_children, base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
             ),
             'success': True
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     @base.parametrize('name,perm,apply_to_children,grant_admin,access_ok', [
         ('none_fails', 'group.none', 'none', False, False),
         ('read_fails', 'group.read', 'none', False, False),
         ('write_fails', 'group.write', 'none', False, False),
         ('admin_fails', 'group.admin', 'none', False, False),
 
         # with granted perms
         ('none_ok', 'group.none', 'none', True, True),
         ('read_ok', 'group.read', 'none', True, True),
         ('write_ok', 'group.write', 'none', True, True),
         ('admin_ok', 'group.admin', 'none', True, True),
     ])
     def test_api_grant_user_permission_to_repo_group_by_regular_user(
             self, name, perm, apply_to_children, grant_admin, access_ok):
         if grant_admin:
             RepoGroupModel().grant_user_permission(TEST_REPO_GROUP,
                                                    self.TEST_USER_LOGIN,
                                                    'group.admin')
             meta.Session().commit()
 
         id_, params = _build_data(self.apikey_regular,
                                   'grant_user_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm, apply_to_children=apply_to_children)
         response = api_call(self, params)
         if access_ok:
             ret = {
                 'msg': 'Granted perm: `%s` (recursive:%s) for user: `%s` in repo group: `%s`' % (
                     perm, apply_to_children, base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
                 ),
                 'success': True
             }
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         else:
             expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP
             self._compare_error(id_, expected, given=response.body)
 
     def test_api_grant_user_permission_to_repo_group_wrong_permission(self):
         perm = 'haha.no.permission'
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'permission `%s` does not exist' % perm
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoGroupModel, 'grant_user_permission', raise_exception)
     def test_api_grant_user_permission_to_repo_group_exception_when_adding(self):
         perm = 'group.read'
         id_, params = _build_data(self.apikey,
                                   'grant_user_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user: `%s` in repo group: `%s`' % (
             base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
         )
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,apply_to_children', [
         ('none', 'none'),
         ('all', 'all'),
         ('repos', 'repos'),
         ('groups', 'groups'),
     ])
     def test_api_revoke_user_permission_from_repo_group(self, name, apply_to_children):
         RepoGroupModel().grant_user_permission(repo_group=TEST_REPO_GROUP,
                                                user=base.TEST_USER_ADMIN_LOGIN,
                                                perm='group.read',)
         meta.Session().commit()
 
         id_, params = _build_data(self.apikey,
                                   'revoke_user_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
 
         expected = {
             'msg': 'Revoked perm (recursive:%s) for user: `%s` in repo group: `%s`' % (
                 apply_to_children, base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
             ),
             'success': True
         }
         self._compare_ok(id_, expected, given=response.body)
 
     @base.parametrize('name,apply_to_children,grant_admin,access_ok', [
         ('none', 'none', False, False),
         ('all', 'all', False, False),
         ('repos', 'repos', False, False),
         ('groups', 'groups', False, False),
 
         # after granting admin rights
         ('none', 'none', False, False),
         ('all', 'all', False, False),
         ('repos', 'repos', False, False),
         ('groups', 'groups', False, False),
     ])
     def test_api_revoke_user_permission_from_repo_group_by_regular_user(
             self, name, apply_to_children, grant_admin, access_ok):
         RepoGroupModel().grant_user_permission(repo_group=TEST_REPO_GROUP,
                                                user=base.TEST_USER_ADMIN_LOGIN,
                                                perm='group.read',)
         meta.Session().commit()
 
         if grant_admin:
             RepoGroupModel().grant_user_permission(TEST_REPO_GROUP,
                                                    self.TEST_USER_LOGIN,
                                                    'group.admin')
             meta.Session().commit()
 
         id_, params = _build_data(self.apikey_regular,
                                   'revoke_user_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
         if access_ok:
             expected = {
                 'msg': 'Revoked perm (recursive:%s) for user: `%s` in repo group: `%s`' % (
                     apply_to_children, base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
                 ),
                 'success': True
             }
             self._compare_ok(id_, expected, given=response.body)
         else:
             expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP
             self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoGroupModel, 'revoke_user_permission', raise_exception)
     def test_api_revoke_user_permission_from_repo_group_exception_when_adding(self):
         id_, params = _build_data(self.apikey,
                                   'revoke_user_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   userid=base.TEST_USER_ADMIN_LOGIN, )
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user: `%s` in repo group: `%s`' % (
             base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
         )
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,perm,apply_to_children', [
         ('none', 'group.none', 'none'),
         ('read', 'group.read', 'none'),
         ('write', 'group.write', 'none'),
         ('admin', 'group.admin', 'none'),
 
         ('none', 'group.none', 'all'),
         ('read', 'group.read', 'all'),
         ('write', 'group.write', 'all'),
         ('admin', 'group.admin', 'all'),
 
         ('none', 'group.none', 'repos'),
         ('read', 'group.read', 'repos'),
         ('write', 'group.write', 'repos'),
         ('admin', 'group.admin', 'repos'),
 
         ('none', 'group.none', 'groups'),
         ('read', 'group.read', 'groups'),
         ('write', 'group.write', 'groups'),
         ('admin', 'group.admin', 'groups'),
     ])
     def test_api_grant_user_group_permission_to_repo_group(self, name, perm, apply_to_children):
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
 
         ret = {
             'msg': 'Granted perm: `%s` (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                 perm, apply_to_children, TEST_USER_GROUP, TEST_REPO_GROUP
             ),
             'success': True
         }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
 
     @base.parametrize('name,perm,apply_to_children,grant_admin,access_ok', [
         ('none_fails', 'group.none', 'none', False, False),
         ('read_fails', 'group.read', 'none', False, False),
         ('write_fails', 'group.write', 'none', False, False),
         ('admin_fails', 'group.admin', 'none', False, False),
 
         # with granted perms
         ('none_ok', 'group.none', 'none', True, True),
         ('read_ok', 'group.read', 'none', True, True),
         ('write_ok', 'group.write', 'none', True, True),
         ('admin_ok', 'group.admin', 'none', True, True),
     ])
     def test_api_grant_user_group_permission_to_repo_group_by_regular_user(
             self, name, perm, apply_to_children, grant_admin, access_ok):
         if grant_admin:
             RepoGroupModel().grant_user_permission(TEST_REPO_GROUP,
                                                    self.TEST_USER_LOGIN,
                                                    'group.admin')
             meta.Session().commit()
 
         id_, params = _build_data(self.apikey_regular,
                                   'grant_user_group_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
         if access_ok:
             ret = {
                 'msg': 'Granted perm: `%s` (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                     perm, apply_to_children, TEST_USER_GROUP, TEST_REPO_GROUP
                 ),
                 'success': True
             }
             expected = ret
             self._compare_ok(id_, expected, given=response.body)
         else:
             expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP
             self._compare_error(id_, expected, given=response.body)
 
     def test_api_grant_user_group_permission_to_repo_group_wrong_permission(self):
         perm = 'haha.no.permission'
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'permission `%s` does not exist' % perm
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoGroupModel, 'grant_user_group_permission', raise_exception)
     def test_api_grant_user_group_permission_exception_when_adding_to_repo_group(self):
         perm = 'group.read'
         id_, params = _build_data(self.apikey,
                                   'grant_user_group_permission_to_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   perm=perm)
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user group: `%s` in repo group: `%s`' % (
             TEST_USER_GROUP, TEST_REPO_GROUP
         )
         self._compare_error(id_, expected, given=response.body)
 
     @base.parametrize('name,apply_to_children', [
         ('none', 'none'),
         ('all', 'all'),
         ('repos', 'repos'),
         ('groups', 'groups'),
     ])
     def test_api_revoke_user_group_permission_from_repo_group(self, name, apply_to_children):
         RepoGroupModel().grant_user_group_permission(repo_group=TEST_REPO_GROUP,
                                                      group_name=TEST_USER_GROUP,
                                                      perm='group.read',)
         meta.Session().commit()
         id_, params = _build_data(self.apikey,
                                   'revoke_user_group_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
 
         expected = {
             'msg': 'Revoked perm (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                 apply_to_children, TEST_USER_GROUP, TEST_REPO_GROUP
             ),
             'success': True
         }
         self._compare_ok(id_, expected, given=response.body)
 
     @base.parametrize('name,apply_to_children,grant_admin,access_ok', [
         ('none', 'none', False, False),
         ('all', 'all', False, False),
         ('repos', 'repos', False, False),
         ('groups', 'groups', False, False),
 
         # after granting admin rights
         ('none', 'none', False, False),
         ('all', 'all', False, False),
         ('repos', 'repos', False, False),
         ('groups', 'groups', False, False),
     ])
     def test_api_revoke_user_group_permission_from_repo_group_by_regular_user(
             self, name, apply_to_children, grant_admin, access_ok):
         RepoGroupModel().grant_user_permission(repo_group=TEST_REPO_GROUP,
                                                user=base.TEST_USER_ADMIN_LOGIN,
                                                perm='group.read',)
         meta.Session().commit()
 
         if grant_admin:
             RepoGroupModel().grant_user_permission(TEST_REPO_GROUP,
                                                    self.TEST_USER_LOGIN,
                                                    'group.admin')
             meta.Session().commit()
 
         id_, params = _build_data(self.apikey_regular,
                                   'revoke_user_group_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,
                                   apply_to_children=apply_to_children,)
         response = api_call(self, params)
         if access_ok:
             expected = {
                 'msg': 'Revoked perm (recursive:%s) for user group: `%s` in repo group: `%s`' % (
                     apply_to_children, base.TEST_USER_ADMIN_LOGIN, TEST_REPO_GROUP
                 ),
                 'success': True
             }
             self._compare_ok(id_, expected, given=response.body)
         else:
             expected = 'repository group `%s` does not exist' % TEST_REPO_GROUP
             self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(RepoGroupModel, 'revoke_user_group_permission', raise_exception)
     def test_api_revoke_user_group_permission_from_repo_group_exception_when_adding(self):
         id_, params = _build_data(self.apikey, 'revoke_user_group_permission_from_repo_group',
                                   repogroupid=TEST_REPO_GROUP,
                                   usergroupid=TEST_USER_GROUP,)
         response = api_call(self, params)
 
         expected = 'failed to edit permission for user group: `%s` in repo group: `%s`' % (
             TEST_USER_GROUP, TEST_REPO_GROUP
         )
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_gist(self):
         gist = fixture.create_gist()
         gist_id = gist.gist_access_id
         gist_created_on = gist.created_on
         id_, params = _build_data(self.apikey, 'get_gist',
                                   gistid=gist_id, )
         response = api_call(self, params)
 
         expected = {
             'access_id': gist_id,
             'created_on': gist_created_on,
             'description': 'new-gist',
             'expires': -1.0,
             'gist_id': int(gist_id),
             'type': 'public',
             'url': 'http://localhost:80/_admin/gists/%s' % gist_id
         }
 
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_gist_that_does_not_exist(self):
         id_, params = _build_data(self.apikey_regular, 'get_gist',
                                   gistid='12345', )
         response = api_call(self, params)
         expected = 'gist `%s` does not exist' % ('12345',)
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_gist_private_gist_without_permission(self):
         gist = fixture.create_gist()
         gist_id = gist.gist_access_id
         gist_created_on = gist.created_on
         id_, params = _build_data(self.apikey_regular, 'get_gist',
                                   gistid=gist_id, )
         response = api_call(self, params)
 
         expected = 'gist `%s` does not exist' % gist_id
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_gists(self):
         fixture.create_gist()
         fixture.create_gist()
 
         id_, params = _build_data(self.apikey, 'get_gists')
         response = api_call(self, params)
         expected = response.json
         assert len(response.json['result']) == 2
         #self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_gists_regular_user(self):
         # by admin
         fixture.create_gist()
         fixture.create_gist()
 
         # by reg user
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
 
         id_, params = _build_data(self.apikey_regular, 'get_gists')
         response = api_call(self, params)
         expected = response.json
         assert len(response.json['result']) == 3
         #self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_gists_only_for_regular_user(self):
         # by admin
         fixture.create_gist()
         fixture.create_gist()
 
         # by reg user
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
         fixture.create_gist(owner=self.TEST_USER_LOGIN)
 
         id_, params = _build_data(self.apikey, 'get_gists',
                                   userid=self.TEST_USER_LOGIN)
         response = api_call(self, params)
         expected = response.json
         assert len(response.json['result']) == 3
         #self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_gists_regular_user_with_different_userid(self):
         id_, params = _build_data(self.apikey_regular, 'get_gists',
                                   userid=base.TEST_USER_ADMIN_LOGIN)
         response = api_call(self, params)
         expected = 'userid is not the same as your user'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_create_gist(self):
         id_, params = _build_data(self.apikey_regular, 'create_gist',
                                   lifetime=10,
                                   description='foobar-gist',
                                   gist_type='public',
                                   files={'foobar': {'content': 'foo'}})
         response = api_call(self, params)
         expected = {
             'gist': {
                 'access_id': response.json['result']['gist']['access_id'],
                 'created_on': response.json['result']['gist']['created_on'],
                 'description': 'foobar-gist',
                 'expires': response.json['result']['gist']['expires'],
                 'gist_id': response.json['result']['gist']['gist_id'],
                 'type': 'public',
                 'url': response.json['result']['gist']['url']
             },
             'msg': 'created new gist'
         }
         self._compare_ok(id_, expected, given=response.body)
 
     @mock.patch.object(GistModel, 'create', raise_exception)
     def test_api_create_gist_exception_occurred(self):
         id_, params = _build_data(self.apikey_regular, 'create_gist',
                                   files={})
         response = api_call(self, params)
         expected = 'failed to create gist'
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_delete_gist(self):
         gist_id = fixture.create_gist().gist_access_id
         id_, params = _build_data(self.apikey, 'delete_gist',
                                   gistid=gist_id)
         response = api_call(self, params)
         expected = {'gist': None, 'msg': 'deleted gist ID:%s' % gist_id}
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_delete_gist_regular_user(self):
         gist_id = fixture.create_gist(owner=self.TEST_USER_LOGIN).gist_access_id
         id_, params = _build_data(self.apikey_regular, 'delete_gist',
                                   gistid=gist_id)
         response = api_call(self, params)
         expected = {'gist': None, 'msg': 'deleted gist ID:%s' % gist_id}
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_delete_gist_regular_user_no_permission(self):
         gist_id = fixture.create_gist().gist_access_id
         id_, params = _build_data(self.apikey_regular, 'delete_gist',
                                   gistid=gist_id)
         response = api_call(self, params)
         expected = 'gist `%s` does not exist' % (gist_id,)
         self._compare_error(id_, expected, given=response.body)
 
     @mock.patch.object(GistModel, 'delete', raise_exception)
     def test_api_delete_gist_exception_occurred(self):
         gist_id = fixture.create_gist().gist_access_id
         id_, params = _build_data(self.apikey, 'delete_gist',
                                   gistid=gist_id)
         response = api_call(self, params)
         expected = 'failed to delete gist ID:%s' % (gist_id,)
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_ip(self):
         id_, params = _build_data(self.apikey, 'get_ip')
         response = api_call(self, params)
         expected = {
             'server_ip_addr': '0.0.0.0',
             'user_ips': []
         }
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_server_info(self):
         id_, params = _build_data(self.apikey, 'get_server_info')
         response = api_call(self, params)
         expected = db.Setting.get_server_info()
         self._compare_ok(id_, expected, given=response.body)
 
     def test_api_get_changesets(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start=0, end=2)
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert len(result) == 3
         assert 'message' in result[0]
         assert 'added' not in result[0]
 
     def test_api_get_changesets_with_max_revisions(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start_date="2011-02-24T00:00:00", max_revisions=10)
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert len(result) == 10
         assert 'message' in result[0]
         assert 'added' not in result[0]
 
     def test_api_get_changesets_with_branch(self):
         if self.REPO == 'vcs_test_hg':
             branch = 'stable'
         else:
             pytest.skip("skipping due to missing branches in git test repo")
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, branch_name=branch, start_date="2011-02-24T00:00:00")
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert len(result) == 5
         assert 'message' in result[0]
         assert 'added' not in result[0]
 
     def test_api_get_changesets_with_file_list(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start_date="2010-04-07T23:30:30", end_date="2010-04-08T00:31:14", with_file_list=True)
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert len(result) == 3
         assert 'message' in result[0]
         assert 'added' in result[0]
 
     def test_api_get_changeset(self):
         review = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         id_, params = _build_data(self.apikey, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION)
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert result["raw_id"] == self.TEST_REVISION
         assert "reviews" not in result
 
     def test_api_get_changeset_with_reviews(self):
         reviewobjs = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         id_, params = _build_data(self.apikey, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION,
                                   with_reviews=True)
         response = api_call(self, params)
         result = ext_json.loads(response.body)["result"]
         assert result["raw_id"] == self.TEST_REVISION
         assert "reviews" in result
         assert len(result["reviews"]) == 1
         review = result["reviews"][0]
         expected = {
             'status': 'approved',
             'modified_at': reviewobjs[0].modified_at.replace(microsecond=0).isoformat(),
             'reviewer': 'test_admin',
         }
         assert review == expected
 
     def test_api_get_changeset_that_does_not_exist(self):
         """ Fetch changeset status for non-existant changeset.
         revision id is the above git hash used in the test above with the
         last 3 nibbles replaced with 0xf.  Should not exist for git _or_ hg.
         """
         id_, params = _build_data(self.apikey, 'get_changeset',
                                   repoid=self.REPO, raw_id = '7ab37bc680b4aa72c34d07b230c866c28e9fcfff')
         response = api_call(self, params)
         expected = 'Changeset %s does not exist' % ('7ab37bc680b4aa72c34d07b230c866c28e9fcfff',)
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_changeset_without_permission(self):
         review = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         RepoModel().revoke_user_permission(repo=self.REPO, user=self.TEST_USER_LOGIN)
         RepoModel().revoke_user_permission(repo=self.REPO, user="default")
         id_, params = _build_data(self.apikey_regular, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION)
         response = api_call(self, params)
         expected = 'Access denied to repo %s' % self.REPO
         self._compare_error(id_, expected, given=response.body)
 
     def test_api_get_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'get test')
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": 'get_pullrequest',
             "args": {"pullrequest_id": pull_request_id},
         }))
         response = api_call(self, params)
         pullrequest = db.PullRequest().get(pull_request_id)
         expected = {
             "status": "new",
             "pull_request_id": pull_request_id,
             "description": "No description",
             "url": "/%s/pull-request/%s/_/%s" % (self.REPO, pull_request_id, "stable"),
             "reviewers": [{"username": "test_regular"}],
             "org_repo_url": "http://localhost:80/%s" % self.REPO,
             "org_ref_parts": ["branch", "stable", self.TEST_PR_SRC],
             "other_ref_parts": ["branch", "default", self.TEST_PR_DST],
             "comments": [{"username": base.TEST_USER_ADMIN_LOGIN, "text": "",
                          "comment_id": pullrequest.comments[0].comment_id}],
             "owner": base.TEST_USER_ADMIN_LOGIN,
             "statuses": [{"status": "under_review", "reviewer": base.TEST_USER_ADMIN_LOGIN, "modified_at": "2000-01-01T00:00:00"} for i in range(0, len(self.TEST_PR_REVISIONS))],
             "title": "get test",
             "revisions": self.TEST_PR_REVISIONS,
             "created_on": "2000-01-01T00:00:00",
             "updated_on": "2000-01-01T00:00:00",
         }
         self._compare_ok(random_id, expected,
                          given=re.sub(br"\d\d\d\d\-\d\d\-\d\dT\d\d\:\d\d\:\d\d",
                                       b"2000-01-01T00:00:00", response.body))
 
     def test_api_close_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'close test')
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "close_pr": True},
         }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = db.PullRequest().get(pull_request_id)
         assert pullrequest.comments[-1].text == ''
         assert pullrequest.status == db.PullRequest.STATUS_CLOSED
         assert pullrequest.is_closed() == True
 
     def test_api_status_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, "status test")
 
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN).api_key,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "status": db.ChangesetStatus.STATUS_APPROVED},
         }))
         response = api_call(self, params)
         pullrequest = db.PullRequest().get(pull_request_id)
         self._compare_error(random_id, "No permission to change pull request status. User needs to be admin, owner or reviewer.", given=response.body)
         assert db.ChangesetStatus.STATUS_UNDER_REVIEW == ChangesetStatusModel().calculate_pull_request_result(pullrequest)[2]
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN).api_key,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "status": db.ChangesetStatus.STATUS_APPROVED},
         }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.ChangesetStatus.STATUS_APPROVED == ChangesetStatusModel().calculate_pull_request_result(pullrequest)[2]
 
     def test_api_comment_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, "comment test")
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "comment_msg": "Looks good to me"},
         }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = db.PullRequest().get(pull_request_id)
         assert pullrequest.comments[-1].text == 'Looks good to me'
 
     def test_api_edit_reviewers_add_single(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "add": base.TEST_USER_REGULAR2_LOGIN},
         }))
         response = api_call(self, params)
         expected = { 'added': [base.TEST_USER_REGULAR2_LOGIN], 'already_present': [], 'removed': [] }
 
         self._compare_ok(random_id, expected, given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_add_nonexistent(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "add": 999},
         }))
         response = api_call(self, params)
 
         self._compare_error(random_id, "user `999` does not exist", given=response.body)
 
     def test_api_edit_reviewers_add_multiple(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {
                 "pull_request_id": pull_request_id,
                 "add": [ self.TEST_USER_LOGIN, base.TEST_USER_REGULAR2_LOGIN ]
             },
         }))
         response = api_call(self, params)
         # list order depends on python sorting hash, which is randomized
         assert set(ext_json.loads(response.body)['result']['added']) == set([base.TEST_USER_REGULAR2_LOGIN, self.TEST_USER_LOGIN])
         assert set(ext_json.loads(response.body)['result']['already_present']) == set()
         assert set(ext_json.loads(response.body)['result']['removed']) == set()
 
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(self.TEST_USER_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_add_already_present(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {
                 "pull_request_id": pull_request_id,
                 "add": [ base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR2_LOGIN ]
             },
         }))
         response = api_call(self, params)
         expected = { 'added': [base.TEST_USER_REGULAR2_LOGIN],
                      'already_present': [base.TEST_USER_REGULAR_LOGIN],
                      'removed': [],
                    }
 
         self._compare_ok(random_id, expected, given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_add_closed(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = self.test_user
         PullRequestModel().close_pull_request(pull_request_id)
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "add": base.TEST_USER_REGULAR2_LOGIN},
         }))
         response = api_call(self, params)
         self._compare_error(random_id, "Cannot edit reviewers of a closed pull request.", given=response.body)
 
     def test_api_edit_reviewers_add_not_owner(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         pullrequest.owner = db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN)
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "add": base.TEST_USER_REGULAR2_LOGIN},
         }))
         response = api_call(self, params)
         self._compare_error(random_id, "No permission to edit reviewers of this pull request. User needs to be admin or pull request owner.", given=response.body)
 
 
     def test_api_edit_reviewers_remove_single(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": base.TEST_USER_REGULAR_LOGIN},
         }))
         response = api_call(self, params)
 
         expected = { 'added': [],
                      'already_present': [],
                      'removed': [base.TEST_USER_REGULAR_LOGIN],
                    }
         self._compare_ok(random_id, expected, given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) not in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_remove_nonexistent(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": 999},
         }))
         response = api_call(self, params)
 
         self._compare_error(random_id, "user `999` does not exist", given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_remove_nonpresent(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) not in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": base.TEST_USER_REGULAR2_LOGIN},
         }))
         response = api_call(self, params)
 
         # NOTE: no explicit indication that removed user was not even a reviewer
         expected = { 'added': [],
                      'already_present': [],
                      'removed': [base.TEST_USER_REGULAR2_LOGIN],
                    }
         self._compare_ok(random_id, expected, given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) not in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_remove_multiple(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         prr = db.PullRequestReviewer(db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN), pullrequest)
         meta.Session().add(prr)
         meta.Session().commit()
 
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": [ base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR2_LOGIN ] },
         }))
         response = api_call(self, params)
 
         # list order depends on python sorting hash, which is randomized
         assert set(ext_json.loads(response.body)['result']['added']) == set()
         assert set(ext_json.loads(response.body)['result']['already_present']) == set()
         assert set(ext_json.loads(response.body)['result']['removed']) == set([base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR2_LOGIN])
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) not in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) not in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_remove_closed(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         PullRequestModel().close_pull_request(pull_request_id)
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": base.TEST_USER_REGULAR_LOGIN},
         }))
         response = api_call(self, params)
 
         self._compare_error(random_id, "Cannot edit reviewers of a closed pull request.", given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_remove_not_owner(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
         pullrequest.owner = db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN)
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id, "remove": base.TEST_USER_REGULAR_LOGIN},
         }))
         response = api_call(self, params)
 
         self._compare_error(random_id, "No permission to edit reviewers of this pull request. User needs to be admin or pull request owner.", given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_add_remove_single(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) not in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id,
                      "add": base.TEST_USER_REGULAR2_LOGIN,
                      "remove": base.TEST_USER_REGULAR_LOGIN
                     },
         }))
         response = api_call(self, params)
 
         expected = { 'added': [base.TEST_USER_REGULAR2_LOGIN],
                      'already_present': [],
                      'removed': [base.TEST_USER_REGULAR_LOGIN],
                    }
         self._compare_ok(random_id, expected, given=response.body)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) not in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_add_remove_multiple(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         prr = db.PullRequestReviewer(db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN), pullrequest)
         meta.Session().add(prr)
         meta.Session().commit()
         assert db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) not in pullrequest.get_reviewer_users()
 
         pullrequest.owner = self.test_user
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id,
                      "add": [ base.TEST_USER_REGULAR2_LOGIN ],
                      "remove": [ base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_ADMIN_LOGIN ],
                     },
         }))
         response = api_call(self, params)
 
         # list order depends on python sorting hash, which is randomized
         assert set(ext_json.loads(response.body)['result']['added']) == set([base.TEST_USER_REGULAR2_LOGIN])
         assert set(ext_json.loads(response.body)['result']['already_present']) == set()
         assert set(ext_json.loads(response.body)['result']['removed']) == set([base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_ADMIN_LOGIN])
         assert db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN) not in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) not in pullrequest.get_reviewer_users()
         assert db.User.get_by_username(base.TEST_USER_REGULAR2_LOGIN) in pullrequest.get_reviewer_users()
 
     def test_api_edit_reviewers_invalid_params(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, 'edit reviewer test')
         pullrequest = db.PullRequest().get(pull_request_id)
         assert db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN) in pullrequest.get_reviewer_users()
 
         pullrequest.owner = db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN)
         random_id = random.randrange(1, 9999)
         params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey_regular,
             "method": "edit_reviewers",
             "args": {"pull_request_id": pull_request_id},
         }))
         response = api_call(self, params)
 
         self._compare_error(random_id, "Invalid request. Neither 'add' nor 'remove' is specified.", given=response.body)
         assert ext_json.loads(response.body)['result'] is None

kallithea/tests/base.py

@@ -1,194 +1,194 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import datetime
 import logging
 import os
 import re
 import tempfile
 import time
 
 import pytest
+from beaker.cache import cache_managers
 from webtest import TestApp
 
 from kallithea.lib.utils2 import ascii_str
 from kallithea.model import db
 
 
 log = logging.getLogger(__name__)
 
 skipif = pytest.mark.skipif
 parametrize = pytest.mark.parametrize
 
 # Hack: These module global values MUST be set to actual values before running any tests. This is currently done by conftest.py.
 url = None
 testapp = None
 
 __all__ = [
     'skipif', 'parametrize', 'url', 'TestController',
     'ldap_lib_installed', 'pam_lib_installed', 'invalidate_all_caches',
     'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'NEW_HG_REPO', 'NEW_GIT_REPO',
     'HG_FORK', 'GIT_FORK', 'TEST_USER_ADMIN_LOGIN', 'TEST_USER_ADMIN_PASS',
     'TEST_USER_ADMIN_EMAIL', 'TEST_USER_REGULAR_LOGIN', 'TEST_USER_REGULAR_PASS',
     'TEST_USER_REGULAR_EMAIL', 'TEST_USER_REGULAR2_LOGIN',
     'TEST_USER_REGULAR2_PASS', 'TEST_USER_REGULAR2_EMAIL', 'IP_ADDR',
     'TEST_HG_REPO', 'TEST_HG_REPO_CLONE', 'TEST_HG_REPO_PULL', 'TEST_GIT_REPO',
     'TEST_GIT_REPO_CLONE', 'TEST_GIT_REPO_PULL', 'HG_REMOTE_REPO',
     'GIT_REMOTE_REPO', 'HG_TEST_REVISION', 'GIT_TEST_REVISION',
 ]
 
 ## SOME GLOBALS FOR TESTS
 
 TESTS_TMP_PATH = os.environ.get('KALLITHEA_TESTS_TMP_PATH', tempfile.mkdtemp(prefix='kallithea-test-'))
 
 TEST_USER_ADMIN_LOGIN = 'test_admin'
 TEST_USER_ADMIN_PASS = 'test12'
 TEST_USER_ADMIN_EMAIL = '[e-mail]'
 
 TEST_USER_REGULAR_LOGIN = 'test_regular'
 TEST_USER_REGULAR_PASS = 'test12'
 TEST_USER_REGULAR_EMAIL = '[e-mail]'
 
 TEST_USER_REGULAR2_LOGIN = 'test_regular2'
 TEST_USER_REGULAR2_PASS = 'test12'
 TEST_USER_REGULAR2_EMAIL = '[e-mail]'
 
 IP_ADDR = '127.0.0.127'
 
 HG_REPO = 'vcs_test_hg'
 GIT_REPO = 'vcs_test_git'
 
 NEW_HG_REPO = 'vcs_test_hg_new'
 NEW_GIT_REPO = 'vcs_test_git_new'
 
 HG_FORK = 'vcs_test_hg_fork'
 GIT_FORK = 'vcs_test_git_fork'
 
 HG_TEST_REVISION = "a53d9201d4bc278910d416d94941b7ea007ecd52"
 GIT_TEST_REVISION = "7ab37bc680b4aa72c34d07b230c866c28e9fc204"
 
 
 ## VCS
 uniq_suffix = str(int(time.mktime(datetime.datetime.now().timetuple())))
 
 GIT_REMOTE_REPO = os.path.join(TESTS_TMP_PATH, GIT_REPO)
 
 TEST_GIT_REPO = os.path.join(TESTS_TMP_PATH, GIT_REPO)
 TEST_GIT_REPO_CLONE = os.path.join(TESTS_TMP_PATH, 'vcs-git-clone-%s' % uniq_suffix)
 TEST_GIT_REPO_PULL = os.path.join(TESTS_TMP_PATH, 'vcs-git-pull-%s' % uniq_suffix)
 
 HG_REMOTE_REPO = os.path.join(TESTS_TMP_PATH, HG_REPO)
 
 TEST_HG_REPO = os.path.join(TESTS_TMP_PATH, HG_REPO)
 TEST_HG_REPO_CLONE = os.path.join(TESTS_TMP_PATH, 'vcs-hg-clone-%s' % uniq_suffix)
 TEST_HG_REPO_PULL = os.path.join(TESTS_TMP_PATH, 'vcs-hg-pull-%s' % uniq_suffix)
 
 # By default, some of the tests will utilise locally available
 # repositories stored within tar.gz archives as source for
 # cloning. Should you wish to use some other, remote archive, simply
 # uncomment these entries and/or update the URLs to use.
 #
 # GIT_REMOTE_REPO = 'git://github.com/codeinn/vcs.git'
 # HG_REMOTE_REPO = 'http://bitbucket.org/marcinkuzminski/vcs'
 
 # skip ldap tests if LDAP lib is not installed
 ldap_lib_installed = False
 try:
     import ldap
     ldap.API_VERSION
     ldap_lib_installed = True
 except ImportError:
     # means that python-ldap is not installed
     pass
 
 try:
     import pam
     pam.PAM_TEXT_INFO
     pam_lib_installed = True
 except ImportError:
     pam_lib_installed = False
 
 
 def invalidate_all_caches():
     """Invalidate all beaker caches currently configured.
     Useful when manipulating IP permissions in a test and changes need to take
     effect immediately.
     Note: Any use of this function is probably a workaround - it should be
     replaced with a more specific cache invalidation in code or test."""
-    from beaker.cache import cache_managers
     for cache in cache_managers.values():
         cache.clear()
 
 
 class NullHandler(logging.Handler):
     def emit(self, record):
         pass
 
 
 class TestController(object):
     """Pytest-style test controller"""
 
     # Note: pytest base classes cannot have an __init__ method
 
     @pytest.fixture(autouse=True)
     def app_fixture(self):
         h = NullHandler()
         logging.getLogger("kallithea").addHandler(h)
         self.app = TestApp(testapp)
         return self.app
 
     def log_user(self, username=TEST_USER_ADMIN_LOGIN,
                  password=TEST_USER_ADMIN_PASS):
         self._logged_username = username
         response = self.app.post(url(controller='login', action='index'),
                                  {'username': username,
                                   'password': password,
                                   '_session_csrf_secret_token': self.session_csrf_secret_token()})
 
         if b'Invalid username or password' in response.body:
             pytest.fail('could not login using %s %s' % (username, password))
 
         assert response.status == '302 Found'
         self.assert_authenticated_user(response, username)
 
         response = response.follow()
         return response.session['authuser']
 
     def _get_logged_user(self):
         return db.User.get_by_username(self._logged_username)
 
     def assert_authenticated_user(self, response, expected_username):
         cookie = response.session.get('authuser')
         user = cookie and cookie.get('user_id')
         user = user and db.User.get(user)
         user = user and user.username
         assert user == expected_username
 
     def session_csrf_secret_token(self):
         return ascii_str(self.app.get(url('session_csrf_secret_token')).body)
 
     def checkSessionFlash(self, response, msg=None, skip=0, _matcher=lambda msg, m: msg in m):
         if 'flash' not in response.session:
             pytest.fail('msg `%s` not found - session has no flash:\n%s' % (msg, response))
         try:
             level, m = response.session['flash'][-1 - skip]
             if _matcher(msg, m):
                 return
         except IndexError:
             pass
         pytest.fail('msg `%s` not found in session flash (skipping %s): %s' %
                     (msg, skip, ', '.join('`%s`' % m for level, m in response.session['flash'])))
 
     def checkSessionFlashRegex(self, response, regex, skip=0):
         self.checkSessionFlash(response, regex, skip=skip, _matcher=re.search)

kallithea/tests/conftest.py

@@ -1,216 +1,216 b''
 import logging
 import os
 import sys
 import time
 
 import formencode
 import pkg_resources
 import pytest
 from paste.deploy import loadwsgi
 from pytest_localserver.http import WSGIServer
 from routes.util import URLGenerator
 from tg.util.webtest import test_context
 
 import kallithea.tests.base  # FIXME: needed for setting testapp instance!!!
 from kallithea.controllers.root import RootController
 from kallithea.lib import inifile
 from kallithea.lib.utils import repo2db_mapper
 from kallithea.model import db, meta
 from kallithea.model.scm import ScmModel
 from kallithea.model.user import UserModel
 from kallithea.tests.base import TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH, invalidate_all_caches
+from kallithea.tests.fixture import create_test_env, create_test_index
 
 
 def pytest_configure():
     os.environ['TZ'] = 'UTC'
     if not kallithea.is_windows:
         time.tzset() # only available on Unix
 
     path = os.getcwd()
     sys.path.insert(0, path)
     pkg_resources.working_set.add_entry(path)
 
     # Disable INFO logging of test database creation, restore with NOTSET
     logging.disable(logging.INFO)
 
     ini_settings = {
         '[server:main]': {
             'port': '4999',
         },
         '[app:main]': {
             'ssh_enabled': 'true',
             # Mainly to safeguard against accidentally overwriting the real one:
             'ssh_authorized_keys': os.path.join(TESTS_TMP_PATH, 'authorized_keys'),
             #'ssh_locale': 'C',
             'app_instance_uuid': 'test',
             'show_revision_number': 'true',
             'session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
             #'i18n.lang': '',
         },
         '[handler_console]': {
             'formatter': 'color_formatter',
         },
         # The 'handler_console_sql' block is very similar to the one in
         # development.ini, but without the explicit 'level=DEBUG' setting:
         # it causes duplicate sqlalchemy debug logs, one through
         # handler_console_sql and another through another path.
         '[handler_console_sql]': {
             'formatter': 'color_formatter_sql',
         },
     }
     create_database = os.environ.get('TEST_DB')  # TODO: rename to 'CREATE_TEST_DB'
     if create_database:
         ini_settings['[app:main]']['sqlalchemy.url'] = create_database
     reuse_database = os.environ.get('REUSE_TEST_DB')
     if reuse_database:
         ini_settings['[app:main]']['sqlalchemy.url'] = reuse_database
 
     test_ini_file = os.path.join(TESTS_TMP_PATH, 'test.ini')
     inifile.create(test_ini_file, None, ini_settings)
 
     context = loadwsgi.loadcontext(loadwsgi.APP, 'config:%s' % test_ini_file)
-    from kallithea.tests.fixture import create_test_env, create_test_index
 
     # set KALLITHEA_NO_TMP_PATH=1 to disable re-creating the database and test repos
     if not int(os.environ.get('KALLITHEA_NO_TMP_PATH', 0)):
         create_test_env(TESTS_TMP_PATH, context.config(), reuse_database=bool(reuse_database))
 
     # set KALLITHEA_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests
     if not int(os.environ.get('KALLITHEA_WHOOSH_TEST_DISABLE', 0)):
         create_test_index(TESTS_TMP_PATH, context.config(), True)
 
     kallithea.tests.base.testapp = context.create()
     # do initial repo scan
     repo2db_mapper(ScmModel().repo_scan(TESTS_TMP_PATH))
 
     logging.disable(logging.NOTSET)
 
     kallithea.tests.base.url = URLGenerator(RootController().mapper, {'HTTP_HOST': 'example.com'})
 
     # set fixed language for form messages, regardless of environment settings
     formencode.api.set_stdtranslation(languages=[])
 
 
 @pytest.fixture
 def create_test_user():
     """Provide users that automatically disappear after test is over."""
     test_user_ids = []
 
     def _create_test_user(user_form):
         user = UserModel().create(user_form)
         test_user_ids.append(user.user_id)
         return user
     yield _create_test_user
     for user_id in test_user_ids:
         UserModel().delete(user_id)
     meta.Session().commit()
 
 
 def _set_settings(*kvtseq):
     session = meta.Session()
     for kvt in kvtseq:
         assert len(kvt) in (2, 3)
         k = kvt[0]
         v = kvt[1]
         t = kvt[2] if len(kvt) == 3 else 'unicode'
         db.Setting.create_or_update(k, v, t)
     session.commit()
 
 
 @pytest.fixture
 def set_test_settings():
     """Restore settings after test is over."""
     # Save settings.
     settings_snapshot = [
         (s.app_settings_name, s.app_settings_value, s.app_settings_type)
         for s in db.Setting.query().all()]
     yield _set_settings
     # Restore settings.
     session = meta.Session()
     keys = frozenset(k for (k, v, t) in settings_snapshot)
     for s in db.Setting.query().all():
         if s.app_settings_name not in keys:
             session.delete(s)
     for k, v, t in settings_snapshot:
         if t == 'list' and hasattr(v, '__iter__'):
             v = ','.join(v) # Quirk: must format list value manually.
         db.Setting.create_or_update(k, v, t)
     session.commit()
 
 
 @pytest.fixture
 def auto_clear_ip_permissions():
     """Fixture that provides nothing but clearing IP permissions upon test
     exit. This clearing is needed to avoid other test failing to make fake http
     accesses."""
     yield
     # cleanup
     user_model = UserModel()
 
     user_ids = []
     user_ids.append(kallithea.DEFAULT_USER_ID)
     user_ids.append(db.User.get_by_username(TEST_USER_REGULAR_LOGIN).user_id)
 
     for user_id in user_ids:
         for ip in db.UserIpMap.query().filter(db.UserIpMap.user_id == user_id):
             user_model.delete_extra_ip(user_id, ip.ip_id)
 
     # IP permissions are cached, need to invalidate this cache explicitly
     invalidate_all_caches()
     session = meta.Session()
     session.commit()
 
 
 @pytest.fixture
 def test_context_fixture(app_fixture):
     """
     Encompass the entire test using this fixture in a test_context,
     making sure that certain functionality still works even if no call to
     self.app.get/post has been made.
     The typical error message indicating you need a test_context is:
         TypeError: No object (name: context) has been registered for this thread
 
     The standard way to fix this is simply using the test_context context
     manager directly inside your test:
         with test_context(self.app):
             <actions>
     but if test setup code (xUnit-style or pytest fixtures) also needs to be
     executed inside the test context, that method is not possible.
     Even if there is no such setup code, the fixture may reduce code complexity
     if the entire test needs to run inside a test context.
 
     To apply this fixture (like any other fixture) to all test methods of a
     class, use the following class decorator:
         @pytest.mark.usefixtures("test_context_fixture")
         class TestFoo(TestController):
             ...
     """
     with test_context(app_fixture):
         yield
 
 
 class MyWSGIServer(WSGIServer):
     def repo_url(self, repo_name, username=TEST_USER_ADMIN_LOGIN, password=TEST_USER_ADMIN_PASS):
         """Return URL to repo on this web server."""
         host, port = self.server_address
         proto = 'http' if self._server.ssl_context is None else 'https'
         auth = ''
         if username is not None:
             auth = username
             if password is not None:
                 auth += ':' + password
         if auth:
             auth += '@'
         return '%s://%s%s:%s/%s' % (proto, auth, host, port, repo_name)
 
 
 @pytest.yield_fixture(scope="session")
 def webserver():
     """Start web server while tests are running.
     Useful for debugging and necessary for vcs operation tests."""
     server = MyWSGIServer(application=kallithea.tests.base.testapp)
     server.start()
 
     yield server
 
     server.stop()

kallithea/tests/fixture.py

@@ -1,441 +1,440 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 Helpers for fixture generation
 """
 
 import logging
 import os
 import shutil
 import tarfile
 from os.path import dirname
 
 from tg import request
 from tg.util.webtest import test_context
 
 from kallithea.lib.auth import AuthUser
 from kallithea.lib.db_manage import DbManage
+from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
+from kallithea.lib.pidlock import DaemonLock
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.model import db, meta
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.comment import ChangesetCommentsModel
 from kallithea.model.gist import GistModel
 from kallithea.model.pull_request import CreatePullRequestAction  # , CreatePullRequestIterationAction, PullRequestModel
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.scm import ScmModel
 from kallithea.model.user import UserModel
 from kallithea.model.user_group import UserGroupModel
 from kallithea.tests.base import (GIT_REPO, HG_REPO, IP_ADDR, TEST_USER_ADMIN_EMAIL, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS, TEST_USER_REGULAR2_EMAIL,
                                   TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                                   TESTS_TMP_PATH, invalidate_all_caches)
+from kallithea.tests.vcs import setup_package
 
 
 log = logging.getLogger(__name__)
 
 FIXTURES = os.path.join(dirname(dirname(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def raise_exception(*args, **kwargs):
     raise Exception('raise_exception raised exception')
 
 
 class Fixture(object):
 
     def __init__(self):
         pass
 
     def anon_access(self, status):
         """
         Context manager for controlling anonymous access.
         Anon access will be set and committed, but restored again when exiting the block.
 
         Usage:
 
         fixture = Fixture()
         with fixture.anon_access(False):
             stuff
         """
 
         class context(object):
             def __enter__(self):
                 anon = db.User.get_default_user()
                 self._before = anon.active
                 anon.active = status
                 meta.Session().commit()
                 invalidate_all_caches()
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = db.User.get_default_user()
                 anon.active = self._before
                 meta.Session().commit()
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         """Return form values to be validated through RepoForm"""
         defs = dict(
             repo_name=None,
             repo_type='hg',
             clone_uri='',
             repo_group='-1',
             repo_description='DESC',
             repo_private=False,
             repo_landing_rev='rev:tip',
             repo_copy_permissions=False,
             repo_state=db.Repository.STATE_CREATED,
         )
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_repo_group_create_params(self, **custom):
         """Return form values to be validated through RepoGroupForm"""
         defs = dict(
             group_name=None,
             group_description='DESC',
             parent_group_id='-1',
             perms_updates=[],
             perms_new=[],
             recursive=False
         )
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = dict(
             username=name,
             password='qweqwe',
             email='[e-mail]' % name,
             firstname='TestUser',
             lastname='Test',
             active=True,
             admin=False,
             extern_type='internal',
             extern_name=None
         )
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = dict(
             users_group_name=name,
             user_group_description='DESC',
             users_group_active=True,
             user_group_data={},
         )
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, repo_group=None, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = db.Repository.get_by_repo_name(name)
             if r:
                 return r
 
         if isinstance(repo_group, db.RepoGroup):
             repo_group = repo_group.group_id
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         form_data['repo_group'] = repo_group # patch form dict so it can be used directly by model
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         meta.Session().commit()
         ScmModel().mark_for_invalidation(name)
         return db.Repository.get_by_repo_name(name)
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = db.Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(repo_name=fork_name,
                                             fork_parent_id=repo_to_fork,
                                             repo_type=repo_to_fork.repo_type,
                                             **kwargs)
         # patch form dict so it can be used directly by model
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_rev']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         meta.Session().commit()
         ScmModel().mark_for_invalidation(fork_name)
         r = db.Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
         RepoModel().delete(repo_name, **kwargs)
         meta.Session().commit()
 
     def create_repo_group(self, name, parent_group_id=None, **kwargs):
         assert '/' not in name, (name, kwargs) # use group_parent_id to make nested groups
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = db.RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_repo_group_create_params(group_name=name, **kwargs)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             parent=parent_group_id,
             owner=kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN),
             )
         meta.Session().commit()
         gr = db.RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         meta.Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = db.User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
         meta.Session().commit()
         user = db.User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         meta.Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = db.UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         meta.Session().commit()
         user_group = db.UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         meta.Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': 'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': db.Gist.GIST_PUBLIC,
             'lifetime': -1,
             'gist_mapping': {'filename1.txt': {'content': 'hello world'}}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'], owner=form_data['owner'], ip_addr=IP_ADDR,
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime']
         )
         meta.Session().commit()
 
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in db.Gist.query():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         meta.Session().commit()
 
     def load_resource(self, resource_name, strip=True):
         with open(os.path.join(FIXTURES, resource_name), 'rb') as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source
 
     def commit_change(self, repo, filename, content, message, vcs_type,
                       parent=None, newfile=False, author=None):
         repo = db.Repository.get_by_repo_name(repo)
         _cs = parent
         if parent is None:
             _cs = EmptyChangeset(alias=vcs_type)
         if author is None:
             author = '%s <%s>' % (TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_EMAIL)
 
         if newfile:
             nodes = {
                 filename: {
                     'content': content
                 }
             }
             cs = ScmModel().create_nodes(
                 user=TEST_USER_ADMIN_LOGIN,
                 ip_addr=IP_ADDR,
                 repo=repo,
                 message=message,
                 nodes=nodes,
                 parent_cs=_cs,
                 author=author,
             )
         else:
             cs = ScmModel().commit_change(
                 repo=repo.scm_instance, repo_name=repo.repo_name,
                 cs=parent,
                 user=TEST_USER_ADMIN_LOGIN,
                 ip_addr=IP_ADDR,
                 author=author,
                 message=message,
                 content=content,
                 f_path=filename
             )
         return cs
 
     def review_changeset(self, repo, revision, status, author=TEST_USER_ADMIN_LOGIN):
         comment = ChangesetCommentsModel().create("review comment", repo, author, revision=revision, send_email=False)
         csm = ChangesetStatusModel().set_status(repo, db.ChangesetStatus.STATUS_APPROVED, author, comment, revision=revision)
         meta.Session().commit()
         return csm
 
     def create_pullrequest(self, testcontroller, repo_name, pr_src_rev, pr_dst_rev, title='title'):
         org_ref = 'branch:stable:%s' % pr_src_rev
         other_ref = 'branch:default:%s' % pr_dst_rev
         with test_context(testcontroller.app): # needed to be able to mock request user and routes.url
             org_repo = other_repo = db.Repository.get_by_repo_name(repo_name)
             owner_user = db.User.get_by_username(TEST_USER_ADMIN_LOGIN)
             reviewers = [db.User.get_by_username(TEST_USER_REGULAR_LOGIN)]
             request.authuser = AuthUser(dbuser=owner_user)
             # creating a PR sends a message with an absolute URL - without routing that requires mocking
             request.environ['routes.url'] = lambda arg, qualified=False, **kwargs: ('https://localhost' if qualified else '') + '/fake/' + arg
             cmd = CreatePullRequestAction(org_repo, other_repo, org_ref, other_ref, title, 'No description', owner_user, reviewers)
             pull_request = cmd.execute()
             meta.Session().commit()
         return pull_request.pull_request_id
 
 
 #==============================================================================
 # Global test environment setup
 #==============================================================================
 
 def create_test_env(repos_test_path, config, reuse_database):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
 
     # PART ONE create db
     dbconf = config['sqlalchemy.url']
     log.debug('making test db %s', dbconf)
 
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s', repos_test_path)
         os.makedirs(repos_test_path)
 
     dbmanage = DbManage(dbconf=dbconf, root=config['here'],
                         cli_args={
                             'force_ask': True,
                             'username': TEST_USER_ADMIN_LOGIN,
                             'password': TEST_USER_ADMIN_PASS,
                             'email': TEST_USER_ADMIN_EMAIL,
                         })
     dbmanage.create_tables(reuse_database=reuse_database)
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.prompt_repo_root_path(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.create_admin_user()
     dbmanage.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, TEST_USER_REGULAR_EMAIL, False)
     dbmanage.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL, False)
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     meta.Session().commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
 
     idx_path = config['index_dir']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     # CREATE DEFAULT TEST REPOS
     tar = tarfile.open(os.path.join(FIXTURES, 'vcs_test_hg.tar.gz'))
     tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
     tar = tarfile.open(os.path.join(FIXTURES, 'vcs_test_git.tar.gz'))
     tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))
     tar.close()
 
     # LOAD VCS test stuff
-    from kallithea.tests.vcs import setup_package
     setup_package()
 
 
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
     """
 
-    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
-    from kallithea.lib.pidlock import DaemonLock
-
     index_location = os.path.join(config['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
 
     l = DaemonLock(os.path.join(index_location, 'make_index.lock'))
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location) \
         .run(full_index=full_index)
     l.release()
 
 
 def failing_test_hook(ui, repo, **kwargs):
     ui.write(b"failing_test_hook failed\n")
     return 1
 
 
 def exception_test_hook(ui, repo, **kwargs):
     raise Exception("exception_test_hook threw an exception")
 
 
 def passing_test_hook(ui, repo, **kwargs):
     ui.write(b"passing_test_hook succeeded\n")
     return 0

kallithea/tests/functional/test_my_account.py

@@ -1,294 +1,293 b''
 # -*- coding: utf-8 -*-
 
 from tg.util.webtest import test_context
 
 from kallithea.lib import webutils
-from kallithea.model import db, meta
+from kallithea.model import db, meta, validators
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 
 
 fixture = Fixture()
 
 
 class TestMyAccountController(base.TestController):
     test_user_1 = 'testme'
 
     @classmethod
     def teardown_class(cls):
         if db.User.get_by_username(cls.test_user_1):
             UserModel().delete(cls.test_user_1)
             meta.Session().commit()
 
     def test_my_account(self):
         self.log_user()
         response = self.app.get(base.url('my_account'))
 
         response.mustcontain('value="%s' % base.TEST_USER_ADMIN_LOGIN)
 
     def test_my_account_my_repos(self):
         self.log_user()
         response = self.app.get(base.url('my_account_repos'))
         cnt = db.Repository.query().filter(db.Repository.owner ==
                            db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)).count()
         response.mustcontain('"raw_name": "%s"' % base.HG_REPO)
         response.mustcontain('"just_name": "%s"' % base.GIT_REPO)
 
     def test_my_account_my_watched(self):
         self.log_user()
         response = self.app.get(base.url('my_account_watched'))
 
         cnt = db.UserFollowing.query().filter(db.UserFollowing.user ==
                             db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)).count()
         response.mustcontain('"raw_name": "%s"' % base.HG_REPO)
         response.mustcontain('"just_name": "%s"' % base.GIT_REPO)
 
     def test_my_account_my_emails(self):
         self.log_user()
         response = self.app.get(base.url('my_account_emails'))
         response.mustcontain('No additional emails specified')
 
     def test_my_account_my_emails_add_existing_email(self):
         self.log_user()
         response = self.app.get(base.url('my_account_emails'))
         response.mustcontain('No additional emails specified')
         response = self.app.post(base.url('my_account_emails'),
                                  {'new_email': base.TEST_USER_REGULAR_EMAIL, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'This email address is already in use')
 
     def test_my_account_my_emails_add_missing_email_in_form(self):
         self.log_user()
         response = self.app.get(base.url('my_account_emails'))
         response.mustcontain('No additional emails specified')
         response = self.app.post(base.url('my_account_emails'),
             {'_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Please enter an email address')
 
     def test_my_account_my_emails_add_remove(self):
         self.log_user()
         response = self.app.get(base.url('my_account_emails'))
         response.mustcontain('No additional emails specified')
 
         response = self.app.post(base.url('my_account_emails'),
                                  {'new_email': '[e-mail]', '_session_csrf_secret_token': self.session_csrf_secret_token()})
 
         response = self.app.get(base.url('my_account_emails'))
 
         email_id = db.UserEmailMap.query() \
             .filter(db.UserEmailMap.user == db.User.get_by_username(base.TEST_USER_ADMIN_LOGIN)) \
             .filter(db.UserEmailMap.email == '[e-mail]').one().email_id
 
         response.mustcontain('[e-mail]')
         response.mustcontain('<input id="del_email_id" name="del_email_id" type="hidden" value="%s" />' % email_id)
 
         response = self.app.post(base.url('my_account_emails_delete'),
                                  {'del_email_id': email_id, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Removed email from user')
         response = self.app.get(base.url('my_account_emails'))
         response.mustcontain('No additional emails specified')
 
 
     @base.parametrize('name,attrs',
         [('firstname', {'firstname': 'new_username'}),
          ('lastname', {'lastname': 'new_username'}),
          ('admin', {'admin': True}),
          ('admin', {'admin': False}),
          ('extern_type', {'extern_type': 'ldap'}),
          ('extern_type', {'extern_type': None}),
          #('extern_name', {'extern_name': 'test'}),
          #('extern_name', {'extern_name': None}),
          ('active', {'active': False}),
          ('active', {'active': True}),
          ('email', {'email': '[e-mail]'}),
         # ('new_password', {'new_password': 'foobar123',
         #                   'password_confirmation': 'foobar123'})
         ])
     def test_my_account_update(self, name, attrs):
         usr = fixture.create_user(self.test_user_1, password='qweqwe',
                                   email='[e-mail]',
                                   extern_type='internal',
                                   extern_name=self.test_user_1,
                                   skip_if_exists=True)
         params = usr.get_api_data(True)  # current user data
         user_id = usr.user_id
         self.log_user(username=self.test_user_1, password='qweqwe')
 
         params.update({'password_confirmation': ''})
         params.update({'new_password': ''})
         params.update({'extern_type': 'internal'})
         params.update({'extern_name': self.test_user_1})
         params.update({'_session_csrf_secret_token': self.session_csrf_secret_token()})
 
         params.update(attrs)
         response = self.app.post(base.url('my_account'), params)
 
         self.checkSessionFlash(response,
                                'Your account was updated successfully')
 
         updated_user = db.User.get_by_username(self.test_user_1)
         updated_params = updated_user.get_api_data(True)
         updated_params.update({'password_confirmation': ''})
         updated_params.update({'new_password': ''})
 
         params['last_login'] = updated_params['last_login']
         if name == 'email':
             params['emails'] = [attrs['email']]
         if name == 'extern_type':
             # cannot update this via form, expected value is original one
             params['extern_type'] = "internal"
         if name == 'extern_name':
             # cannot update this via form, expected value is original one
             params['extern_name'] = str(user_id)
         if name == 'active':
             # my account cannot deactivate account
             params['active'] = True
         if name == 'admin':
             # my account cannot make you an admin !
             params['admin'] = False
 
         params.pop('_session_csrf_secret_token')
         assert params == updated_params
 
     def test_my_account_update_err_email_exists(self):
         self.log_user()
 
         new_email = base.TEST_USER_REGULAR_EMAIL  # already existing email
         response = self.app.post(base.url('my_account'),
                                 params=dict(
                                     username=base.TEST_USER_ADMIN_LOGIN,
                                     new_password=base.TEST_USER_ADMIN_PASS,
                                     password_confirmation='test122',
                                     firstname='NewName',
                                     lastname='NewLastname',
                                     email=new_email,
                                     _session_csrf_secret_token=self.session_csrf_secret_token())
                                 )
 
         response.mustcontain('This email address is already in use')
 
     def test_my_account_update_err(self):
         self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
 
         new_email = 'newmail.pl'
         response = self.app.post(base.url('my_account'),
                                  params=dict(
                                             username=base.TEST_USER_ADMIN_LOGIN,
                                             new_password=base.TEST_USER_ADMIN_PASS,
                                             password_confirmation='test122',
                                             firstname='NewName',
                                             lastname='NewLastname',
                                             email=new_email,
                                             _session_csrf_secret_token=self.session_csrf_secret_token()))
 
         response.mustcontain('An email address must contain a single @')
-        from kallithea.model import validators
         with test_context(self.app):
             msg = validators.ValidUsername(edit=False, old_data={}) \
                     ._messages['username_exists']
         msg = webutils.html_escape(msg % {'username': base.TEST_USER_ADMIN_LOGIN})
         response.mustcontain(msg)
 
     def test_my_account_api_keys(self):
         usr = self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         user = db.User.get(usr['user_id'])
         response = self.app.get(base.url('my_account_api_keys'))
         response.mustcontain(user.api_key)
         response.mustcontain('Expires: Never')
 
     @base.parametrize('desc,lifetime', [
         ('forever', -1),
         ('5mins', 60*5),
         ('30days', 60*60*24*30),
     ])
     def test_my_account_add_api_keys(self, desc, lifetime):
         usr = self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         user = db.User.get(usr['user_id'])
         response = self.app.post(base.url('my_account_api_keys'),
                                  {'description': desc, 'lifetime': lifetime, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'API key successfully created')
         try:
             response = response.follow()
             user = db.User.get(usr['user_id'])
             for api_key in user.api_keys:
                 response.mustcontain(api_key)
         finally:
             for api_key in db.UserApiKeys.query().all():
                 meta.Session().delete(api_key)
                 meta.Session().commit()
 
     def test_my_account_remove_api_key(self):
         usr = self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         user = db.User.get(usr['user_id'])
         response = self.app.post(base.url('my_account_api_keys'),
                                  {'description': 'desc', 'lifetime': -1, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'API key successfully created')
         response = response.follow()
 
         # now delete our key
         keys = db.UserApiKeys.query().all()
         assert 1 == len(keys)
 
         response = self.app.post(base.url('my_account_api_keys_delete'),
                  {'del_api_key': keys[0].api_key, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'API key successfully deleted')
         keys = db.UserApiKeys.query().all()
         assert 0 == len(keys)
 
     def test_my_account_reset_main_api_key(self):
         usr = self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         user = db.User.get(usr['user_id'])
         api_key = user.api_key
         response = self.app.get(base.url('my_account_api_keys'))
         response.mustcontain(api_key)
         response.mustcontain('Expires: Never')
 
         response = self.app.post(base.url('my_account_api_keys_delete'),
                  {'del_api_key_builtin': api_key, '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'API key successfully reset')
         response = response.follow()
         response.mustcontain(no=[api_key])
 
     def test_my_account_add_ssh_key(self):
         description = 'something'
         public_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUQ== me@localhost'
         fingerprint = 'Ke3oUCNJM87P0jJTb3D+e3shjceP2CqMpQKVd75E9I8'
 
         self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         response = self.app.post(base.url('my_account_ssh_keys'),
                                  {'description': description,
                                   'public_key': public_key,
                                   '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'SSH key %s successfully added' % fingerprint)
 
         response = response.follow()
         response.mustcontain(fingerprint)
         user_id = response.session['authuser']['user_id']
         ssh_key = db.UserSshKeys.query().filter(db.UserSshKeys.user_id == user_id).one()
         assert ssh_key.fingerprint == fingerprint
         assert ssh_key.description == description
         meta.Session().delete(ssh_key)
         meta.Session().commit()
 
     def test_my_account_remove_ssh_key(self):
         description = ''
         public_key = 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUQ== me@localhost'
         fingerprint = 'Ke3oUCNJM87P0jJTb3D+e3shjceP2CqMpQKVd75E9I8'
 
         self.log_user(base.TEST_USER_REGULAR2_LOGIN, base.TEST_USER_REGULAR2_PASS)
         response = self.app.post(base.url('my_account_ssh_keys'),
                                  {'description': description,
                                   'public_key': public_key,
                                   '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'SSH key %s successfully added' % fingerprint)
         response.follow()
         user_id = response.session['authuser']['user_id']
         ssh_key = db.UserSshKeys.query().filter(db.UserSshKeys.user_id == user_id).one()
         assert ssh_key.description == 'me@localhost'
 
         response = self.app.post(base.url('my_account_ssh_keys_delete'),
                                  {'del_public_key_fingerprint': ssh_key.fingerprint,
                                   '_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'SSH key successfully deleted')
         keys = db.UserSshKeys.query().all()
         assert 0 == len(keys)

kallithea/tests/other/test_libs.py

@@ -1,592 +1,580 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.other.test_libs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Package for testing various lib/helper functions in kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 9, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import datetime
 import hashlib
+import re
 
 import mock
+import routes
+from dateutil import relativedelta
+from tg import request
 from tg.util.webtest import test_context
 
 import kallithea.lib.helpers as h
 from kallithea.lib import webutils
-from kallithea.lib.utils2 import AttributeDict, safe_bytes
+from kallithea.lib.utils2 import AttributeDict, get_clone_url, safe_bytes
 from kallithea.model import db
 from kallithea.tests import base
 
 
 proto = 'http'
 TEST_URLS = [
     ('%s://127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://username@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://username:pass@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://127.0.0.1:8080' % proto, ['%s://' % proto, '127.0.0.1', '8080'],
      '%s://127.0.0.1:8080' % proto),
     ('%s://example.com' % proto, ['%s://' % proto, 'example.com'],
      '%s://example.com' % proto),
     ('%s://user:[e-mail]:8080' % proto, ['%s://' % proto, 'example.com',
                                                 '8080'],
      '%s://example.com:8080' % proto),
 ]
 
 proto = 'https'
 TEST_URLS += [
     ('%s://127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://username@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://username:pass@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
      '%s://127.0.0.1' % proto),
     ('%s://127.0.0.1:8080' % proto, ['%s://' % proto, '127.0.0.1', '8080'],
      '%s://127.0.0.1:8080' % proto),
     ('%s://example.com' % proto, ['%s://' % proto, 'example.com'],
      '%s://example.com' % proto),
     ('%s://user:[e-mail]:8080' % proto, ['%s://' % proto, 'example.com',
                                                 '8080'],
      '%s://example.com:8080' % proto),
 ]
 
 
 class TestLibs(base.TestController):
 
     @base.parametrize('test_url,expected,expected_creds', TEST_URLS)
     def test_uri_filter(self, test_url, expected, expected_creds):
         from kallithea.lib.utils2 import uri_filter
         assert uri_filter(test_url) == expected
 
     @base.parametrize('test_url,expected,expected_creds', TEST_URLS)
     def test_credentials_filter(self, test_url, expected, expected_creds):
         from kallithea.lib.utils2 import credentials_filter
         assert credentials_filter(test_url) == expected_creds
 
     @base.parametrize('str_bool,expected', [
                            ('t', True),
                            ('true', True),
                            ('y', True),
                            ('yes', True),
                            ('on', True),
                            ('1', True),
                            ('Y', True),
                            ('yeS', True),
                            ('Y', True),
                            ('TRUE', True),
                            ('T', True),
                            ('False', False),
                            ('F', False),
                            ('FALSE', False),
                            ('0', False),
     ])
     def test_asbool(self, str_bool, expected):
         from kallithea.lib.utils2 import asbool
         assert asbool(str_bool) == expected
 
     def test_mention_extractor(self):
         from kallithea.lib.utils2 import extract_mentioned_usernames
         sample = (
             "@first hi there @world here's my email [e-mail] "
             "@lukaszb check @one_more22 it pls @ ttwelve @D[] @one@two@three "
             "@UPPER    @cAmEL @2one_more22 @john please see this http://org.pl "
             "@marian.user just do it @marco-polo and next extract @marco_polo "
             "user.dot  hej ! not-needed [e-mail]"
         )
 
         expected = set([
             '2one_more22', 'first', 'lukaszb', 'one', 'one_more22', 'UPPER', 'cAmEL', 'john',
             'marian.user', 'marco-polo', 'marco_polo', 'world'])
         assert expected == set(extract_mentioned_usernames(sample))
 
     @base.parametrize('age_args,expected', [
         (dict(), 'just now'),
         (dict(seconds= -1), '1 second ago'),
         (dict(seconds= -60 * 2), '2 minutes ago'),
         (dict(hours= -1), '1 hour ago'),
         (dict(hours= -24), '1 day ago'),
         (dict(hours= -24 * 5), '5 days ago'),
         (dict(months= -1), '1 month ago'),
         (dict(months= -1, days= -2), '1 month and 2 days ago'),
         (dict(months= -1, days= -20), '1 month and 19 days ago'),
         (dict(years= -1, months= -1), '1 year and 1 month ago'),
         (dict(years= -1, months= -10), '1 year and 10 months ago'),
         (dict(years= -2, months= -4), '2 years and 4 months ago'),
         (dict(years= -2, months= -11), '2 years and 11 months ago'),
         (dict(years= -3, months= -2), '3 years and 2 months ago'),
     ])
     def test_age(self, age_args, expected):
-        from dateutil import relativedelta
-
         from kallithea.lib.utils2 import age
         with test_context(self.app):
             n = datetime.datetime(year=2012, month=5, day=17)
             delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
             assert age(n + delt(**age_args), now=n) == expected
 
     @base.parametrize('age_args,expected', [
         (dict(), 'just now'),
         (dict(seconds= -1), '1 second ago'),
         (dict(seconds= -60 * 2), '2 minutes ago'),
         (dict(hours= -1), '1 hour ago'),
         (dict(hours= -24), '1 day ago'),
         (dict(hours= -24 * 5), '5 days ago'),
         (dict(months= -1), '1 month ago'),
         (dict(months= -1, days= -2), '1 month ago'),
         (dict(months= -1, days= -20), '1 month ago'),
         (dict(years= -1, months= -1), '13 months ago'),
         (dict(years= -1, months= -10), '22 months ago'),
         (dict(years= -2, months= -4), '2 years ago'),
         (dict(years= -2, months= -11), '3 years ago'),
         (dict(years= -3, months= -2), '3 years ago'),
         (dict(years= -4, months= -8), '5 years ago'),
     ])
     def test_age_short(self, age_args, expected):
-        from dateutil import relativedelta
-
         from kallithea.lib.utils2 import age
         with test_context(self.app):
             n = datetime.datetime(year=2012, month=5, day=17)
             delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
             assert age(n + delt(**age_args), show_short_version=True, now=n) == expected
 
     @base.parametrize('age_args,expected', [
         (dict(), 'just now'),
         (dict(seconds=1), 'in 1 second'),
         (dict(seconds=60 * 2), 'in 2 minutes'),
         (dict(hours=1), 'in 1 hour'),
         (dict(hours=24), 'in 1 day'),
         (dict(hours=24 * 5), 'in 5 days'),
         (dict(months=1), 'in 1 month'),
         (dict(months=1, days=1), 'in 1 month and 1 day'),
         (dict(years=1, months=1), 'in 1 year and 1 month')
     ])
     def test_age_in_future(self, age_args, expected):
-        from dateutil import relativedelta
-
         from kallithea.lib.utils2 import age
         with test_context(self.app):
             n = datetime.datetime(year=2012, month=5, day=17)
             delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
             assert age(n + delt(**age_args), now=n) == expected
 
     def test_tag_extractor(self):
         sample = (
             "hello pta[tag] gog [[]] [[] sda ero[or]d [me =>>< sa]"
             "[requires] [stale] [see<>=>] [see => http://example.com]"
             "[requires => url] [lang => python] [just a tag]"
             "[,d] [ => ULR ] [obsolete] [desc]]"
         )
         res = h.urlify_text(sample, stylize=True)
         assert '<div class="label label-meta" data-tag="tag">tag</div>' in res
         assert '<div class="label label-meta" data-tag="obsolete">obsolete</div>' in res
         assert '<div class="label label-meta" data-tag="stale">stale</div>' in res
         assert '<div class="label label-meta" data-tag="lang">python</div>' in res
         assert '<div class="label label-meta" data-tag="requires">requires =&gt; <a href="/url">url</a></div>' in res
         assert '<div class="label label-meta" data-tag="tag">tag</div>' in res
 
     def test_alternative_gravatar(self):
         _md5 = lambda s: hashlib.md5(safe_bytes(s)).hexdigest()
 
         # mock tg.tmpl_context
         def fake_tmpl_context(_url):
             _c = AttributeDict()
             _c.visual = AttributeDict()
             _c.visual.use_gravatar = True
             _c.visual.gravatar_url = _url
 
             return _c
 
         with mock.patch('kallithea.lib.webutils.url.current', lambda *a, **b: 'https://example.com'):
             fake = fake_tmpl_context(_url='http://example.com/{email}')
-            with mock.patch('tg.tmpl_context', fake):
-                    from kallithea.lib.webutils import url
-                    assert url.current() == 'https://example.com'
+            with mock.patch('kallithea.lib.helpers.c', fake):
+                    assert webutils.url.current() == 'https://example.com'
                     grav = h.gravatar_url(email_address='[e-mail]', size=24)
                     assert grav == 'http://example.com/[e-mail]'
 
             fake = fake_tmpl_context(_url='http://example.com/{email}')
-            with mock.patch('tg.tmpl_context', fake):
+            with mock.patch('kallithea.lib.helpers.c', fake):
                 grav = h.gravatar_url(email_address='[e-mail]', size=24)
                 assert grav == 'http://example.com/[e-mail]'
 
             fake = fake_tmpl_context(_url='http://example.com/{md5email}')
-            with mock.patch('tg.tmpl_context', fake):
+            with mock.patch('kallithea.lib.helpers.c', fake):
                 em = '[e-mail]'
                 grav = h.gravatar_url(email_address=em, size=24)
                 assert grav == 'http://example.com/%s' % (_md5(em))
 
             fake = fake_tmpl_context(_url='http://example.com/{md5email}/{size}')
-            with mock.patch('tg.tmpl_context', fake):
+            with mock.patch('kallithea.lib.helpers.c', fake):
                 em = '[e-mail]'
                 grav = h.gravatar_url(email_address=em, size=24)
                 assert grav == 'http://example.com/%s/%s' % (_md5(em), 24)
 
             fake = fake_tmpl_context(_url='{scheme}://{netloc}/{md5email}/{size}')
-            with mock.patch('tg.tmpl_context', fake):
+            with mock.patch('kallithea.lib.helpers.c', fake):
                 em = '[e-mail]'
                 grav = h.gravatar_url(email_address=em, size=24)
                 assert grav == 'https://example.com/%s/%s' % (_md5(em), 24)
 
     @base.parametrize('clone_uri_tmpl,repo_name,username,prefix,expected', [
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', None, '', 'http://vps1:8000/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', 'username', '', 'http://username@vps1:8000/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', None, '/prefix', 'http://vps1:8000/prefix/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', 'user', '/prefix', 'http://user@vps1:8000/prefix/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', 'username', '/prefix', 'http://username@vps1:8000/prefix/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', 'user', '/prefix/', 'http://user@vps1:8000/prefix/group/repo1'),
         (db.Repository.DEFAULT_CLONE_URI, 'group/repo1', 'username', '/prefix/', 'http://username@vps1:8000/prefix/group/repo1'),
         ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', None, '', 'http://vps1:8000/_23'),
         ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', 'username', '', 'http://username@vps1:8000/_23'),
         ('http://{user}@{netloc}/_{repoid}', 'group/repo1', 'username', '', 'http://username@vps1:8000/_23'),
         ('http://{netloc}/_{repoid}', 'group/repo1', 'username', '', 'http://vps1:8000/_23'),
         ('https://{user}@proxy1.example.com/{repo}', 'group/repo1', 'username', '', 'https://[e-mail]/group/repo1'),
         ('https://{user}@proxy1.example.com/{repo}', 'group/repo1', None, '', 'https://proxy1.example.com/group/repo1'),
         ('https://proxy1.example.com/{user}/{repo}', 'group/repo1', 'username', '', 'https://proxy1.example.com/username/group/repo1'),
     ])
     def test_clone_url_generator(self, clone_uri_tmpl, repo_name, username, prefix, expected):
-        from kallithea.lib.utils2 import get_clone_url
         clone_url = get_clone_url(clone_uri_tmpl=clone_uri_tmpl, prefix_url='http://vps1:8000' + prefix,
                                   repo_name=repo_name, repo_id=23, username=username)
         assert clone_url == expected
 
     def _quick_url(self, text, tmpl="""<a class="changeset_hash" href="%s">%s</a>""", url_=None):
         """
         Changes `some text url[foo]` => `some text <a href="/">foo</a>
 
         :param text:
         """
-        import re
-
         # quickly change expected url[] into a link
         url_pattern = re.compile(r'(?:url\[)(.+?)(?:\])')
 
         def url_func(match_obj):
             _url = match_obj.groups()[0]
             return tmpl % (url_ or '/repo_name/changeset/%s' % _url, _url)
         return url_pattern.sub(url_func, text)
 
     @base.parametrize('sample,expected', [
       ("",
        ""),
       ("git-svn-id: https://svn.apache.org/repos/asf/libcloud/trunk@1441655 13f79535-47bb-0310-9956-ffa450edef68",
        """git-svn-id: <a href="https://svn.apache.org/repos/asf/libcloud/trunk@1441655">https://svn.apache.org/repos/asf/libcloud/trunk@1441655</a> 13f79535-47bb-0310-9956-ffa450edef68"""),
       ("from rev 000000000000",
        """from rev url[000000000000]"""),
       ("from rev 000000000000123123 also rev 000000000000",
        """from rev url[000000000000123123] also rev url[000000000000]"""),
       ("this should-000 00",
        """this should-000 00"""),
       ("longtextffffffffff rev 123123123123",
        """longtextffffffffff rev url[123123123123]"""),
       ("rev ffffffffffffffffffffffffffffffffffffffffffffffffff",
        """rev ffffffffffffffffffffffffffffffffffffffffffffffffff"""),
       ("ffffffffffff some text traalaa",
        """url[ffffffffffff] some text traalaa"""),
        ("""Multi line
        123123123123
        some text 123123123123
        sometimes !
        """,
        """Multi line<br/>"""
        """       url[123123123123]<br/>"""
        """       some text url[123123123123]<br/>"""
        """       sometimes !"""),
     ])
     def test_urlify_text(self, sample, expected):
         expected = self._quick_url(expected)
 
         with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__',
             lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs,
         ):
             assert h.urlify_text(sample, 'repo_name') == expected
 
     @base.parametrize('sample,expected,url_', [
       ("",
        "",
        ""),
       ("https://svn.apache.org/repos",
        """url[https://svn.apache.org/repos]""",
        "https://svn.apache.org/repos"),
       ("http://svn.apache.org/repos",
        """url[http://svn.apache.org/repos]""",
        "http://svn.apache.org/repos"),
       ("from rev a also rev http://google.com",
        """from rev a also rev url[http://google.com]""",
        "http://google.com"),
       ("http://imgur.com/foo.gif inline http://imgur.com/foo.gif ending http://imgur.com/foo.gif",
        """url[http://imgur.com/foo.gif] inline url[http://imgur.com/foo.gif] ending url[http://imgur.com/foo.gif]""",
        "http://imgur.com/foo.gif"),
       ("""Multi line
        https://foo.bar.example.com
        some text lalala""",
        """Multi line<br/>"""
        """       url[https://foo.bar.example.com]<br/>"""
        """       some text lalala""",
        "https://foo.bar.example.com"),
       ("@mention @someone",
        """<b>@mention</b> <b>@someone</b>""",
        ""),
       ("deadbeefcafe 123412341234",
        """<a class="changeset_hash" href="/repo_name/changeset/deadbeefcafe">deadbeefcafe</a> <a class="changeset_hash" href="/repo_name/changeset/123412341234">123412341234</a>""",
        ""),
       ("We support * markup for *bold* markup of *single or multiple* words, "
        "*a bit @like http://slack.com*. "
        "The first * must come after whitespace and not be followed by whitespace, "
        "contain anything but * and newline until the next *, "
        "which must not come after whitespace "
        "and not be followed by * or alphanumerical *characters*.",
        """We support * markup for <b>*bold*</b> markup of <b>*single or multiple*</b> words, """
        """<b>*a bit <b>@like</b> <a href="http://slack.com">http://slack.com</a>*</b>. """
        """The first * must come after whitespace and not be followed by whitespace, """
        """contain anything but * and newline until the next *, """
        """which must not come after whitespace """
        """and not be followed by * or alphanumerical <b>*characters*</b>.""",
        "-"),
       ("HTML escaping: <abc> 'single' \"double\" &pointer",
        "HTML escaping: &lt;abc&gt; &#39;single&#39; &quot;double&quot; &amp;pointer",
        "-"),
       # tags are covered by test_tag_extractor
     ])
     def test_urlify_test(self, sample, expected, url_):
         expected = self._quick_url(expected,
                                    tmpl="""<a href="%s">%s</a>""", url_=url_)
         with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__',
             lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs,
         ):
             assert h.urlify_text(sample, 'repo_name', stylize=True) == expected
 
     @base.parametrize('sample,expected', [
       ("deadbeefcafe @mention, and http://foo.bar/ yo",
        """<a class="changeset_hash" href="/repo_name/changeset/deadbeefcafe">deadbeefcafe</a>"""
        """<a class="message-link" href="#the-link"> <b>@mention</b>, and </a>"""
        """<a href="http://foo.bar/">http://foo.bar/</a>"""
        """<a class="message-link" href="#the-link"> yo</a>"""),
     ])
     def test_urlify_link(self, sample, expected):
         with mock.patch('kallithea.lib.webutils.UrlGenerator.__call__',
             lambda self, name, **kwargs: dict(changeset_home='/%(repo_name)s/changeset/%(revision)s')[name] % kwargs,
         ):
             assert h.urlify_text(sample, 'repo_name', link_='#the-link') == expected
 
     @base.parametrize('issue_pat,issue_server,issue_sub,sample,expected', [
         (r'#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'issue #123 and issue#456',
             """issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a> and """
             """issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/456">#456</a>"""),
         (r'(?:\s*#)(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'issue #123 and issue#456',
             """issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a> and """
             """issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/456">#456</a>"""),
         # to require whitespace before the issue reference, one may be tempted to use \b...
         (r'\bPR(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'issue PR123 and issuePR456',
             """issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a> and """
             """issuePR456"""),
         # ... but it turns out that \b does not work well in combination with '#': the expectations
         # are reversed from what is actually happening.
         (r'\b#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'issue #123 and issue#456',
             """issue #123 and """
             """issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/456">#456</a>"""),
         # ... so maybe try to be explicit? Unfortunately the whitespace before the issue
         # reference is not retained, again, because it is part of the pattern.
         (r'(?:^|\s)#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             '#15 and issue #123 and issue#456',
             """<a class="issue-tracker-link" href="http://foo/repo_name/issue/15">#15</a> and """
             """issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a> and """
             """issue#456"""),
         # ... instead, use lookbehind assertions.
         (r'(?:^|(?<=\s))#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             '#15 and issue #123 and issue#456',
             """<a class="issue-tracker-link" href="http://foo/repo_name/issue/15">#15</a> and """
             """issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a> and """
             """issue#456"""),
         (r'(?:pullrequest|pull request|PR|pr) ?#?(\d+)', 'http://foo/{repo}/issue/\\1', 'PR#\\1',
             'fixed with pullrequest #1, pull request#2, PR 3, pr4',
             """fixed with <a class="issue-tracker-link" href="http://foo/repo_name/issue/1">PR#1</a>, """
             """<a class="issue-tracker-link" href="http://foo/repo_name/issue/2">PR#2</a>, """
             """<a class="issue-tracker-link" href="http://foo/repo_name/issue/3">PR#3</a>, """
             """<a class="issue-tracker-link" href="http://foo/repo_name/issue/4">PR#4</a>"""),
         (r'#(\d+)', 'http://foo/{repo}/issue/\\1', 'PR\\1',
             'interesting issue #123',
             """interesting issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">PR123</a>"""),
         (r'BUG\d{5}', 'https://bar/{repo}/\\1', '\\1',
             'silly me, I did not parenthesize the id, BUG12345.',
             """silly me, I did not parenthesize the id, <a class="issue-tracker-link" href="https://bar/repo_name/\\1">BUG12345</a>."""),
         (r'BUG(\d{5})', 'https://bar/{repo}/', 'BUG\\1',
             'silly me, the URL does not contain id, BUG12345.',
             """silly me, the URL does not contain id, <a class="issue-tracker-link" href="https://bar/repo_name/">BUG12345</a>."""),
         (r'(PR-\d+)', 'http://foo/{repo}/issue/\\1', '',
             'interesting issue #123, err PR-56',
             """interesting issue #123, err <a class="issue-tracker-link" href="http://foo/repo_name/issue/PR-56">PR-56</a>"""),
         (r'#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             "some 'standard' text with apostrophes",
             """some &#39;standard&#39; text with apostrophes"""),
         (r'#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             "some 'standard' issue #123",
             """some &#39;standard&#39; issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a>"""),
         (r'#(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'an issue   #123       with extra whitespace',
             """an issue   <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a>       with extra whitespace"""),
         (r'(?:\s*#)(\d+)', 'http://foo/{repo}/issue/\\1', '#\\1',
             'an issue   #123       with extra whitespace',
             """an issue<a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a>       with extra whitespace"""),
         # invalid issue pattern
         (r'(PR\d+', 'http://foo/{repo}/issue/{id}', '',
             'PR135',
             """PR135"""),
         # other character than #
         (r'(?:^|(?<=\s))\$(\d+)', 'http://foo/{repo}/issue/\\1', '',
             'empty issue_sub $123 and issue$456',
             """empty issue_sub <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">$123</a> and """
             """issue$456"""),
         # named groups
         (r'(PR|pullrequest|pull request) ?(?P<sitecode>BRU|CPH|BER)-(?P<id>\d+)', r'http://foo/\g<sitecode>/pullrequest/\g<id>/', r'PR-\g<sitecode>-\g<id>',
             'pullrequest CPH-789 is similar to PRBRU-747',
             """<a class="issue-tracker-link" href="http://foo/CPH/pullrequest/789/">PR-CPH-789</a> is similar to """
             """<a class="issue-tracker-link" href="http://foo/BRU/pullrequest/747/">PR-BRU-747</a>"""),
     ])
     def test_urlify_issues(self, issue_pat, issue_server, issue_sub, sample, expected):
         config_stub = {
             'sqlalchemy.url': 'foo',
             'issue_pat': issue_pat,
             'issue_server_link': issue_server,
             'issue_sub': issue_sub,
         }
         # force recreation of lazy function
         with mock.patch('kallithea.lib.helpers._urlify_issues_f', None):
             with mock.patch('kallithea.CONFIG', config_stub):
                 assert h.urlify_text(sample, 'repo_name') == expected
 
     @base.parametrize('sample,expected', [
         ('abc X5', 'abc <a class="issue-tracker-link" href="http://main/repo_name/main/5/">#5</a>'),
         ('abc pullrequest #6 xyz', 'abc <a class="issue-tracker-link" href="http://pr/repo_name/pr/6">PR#6</a> xyz'),
         ('pull request7 #', '<a class="issue-tracker-link" href="http://pr/repo_name/pr/7">PR#7</a> #'),
         ('look PR9 and pr #11', 'look <a class="issue-tracker-link" href="http://pr/repo_name/pr/9">PR#9</a> and <a class="issue-tracker-link" href="http://pr/repo_name/pr/11">PR#11</a>'),
         ('pullrequest#10 solves issue 9', '<a class="issue-tracker-link" href="http://pr/repo_name/pr/10">PR#10</a> solves <a class="issue-tracker-link" href="http://bug/repo_name/bug/9">bug#9</a>'),
         ('issue FAIL67', 'issue <a class="issue-tracker-link" href="http://fail/repo_name/67">FAIL67</a>'),
         ('issue FAILMORE89', 'issue FAILMORE89'), # no match because absent prefix
     ])
     def test_urlify_issues_multiple_issue_patterns(self, sample, expected):
         config_stub = {
             'sqlalchemy.url': r'foo',
             'issue_pat': r'X(\d+)',
             'issue_server_link': r'http://main/{repo}/main/\1/',
             'issue_sub': r'#\1',
             'issue_pat_pr': r'(?:pullrequest|pull request|PR|pr) ?#?(\d+)',
             'issue_server_link_pr': r'http://pr/{repo}/pr/\1',
             'issue_sub_pr': r'PR#\1',
             'issue_pat_bug': r'(?:BUG|bug|issue) ?#?(\d+)',
             'issue_server_link_bug': r'http://bug/{repo}/bug/\1',
             'issue_sub_bug': r'bug#\1',
             'issue_pat_empty_prefix': r'FAIL(\d+)',
             'issue_server_link_empty_prefix': r'http://fail/{repo}/\1',
             'issue_sub_empty_prefix': r'',
             'issue_pat_absent_prefix': r'FAILMORE(\d+)',
             'issue_server_link_absent_prefix': r'http://failmore/{repo}/\1',
         }
         # force recreation of lazy function
         with mock.patch('kallithea.lib.helpers._urlify_issues_f', None):
             with mock.patch('kallithea.CONFIG', config_stub):
                 assert h.urlify_text(sample, 'repo_name') == expected
 
     @base.parametrize('test,expected', [
       ("", None),
       ("/_2", None),
       ("_2", 2),
       ("_2/", None),
     ])
     def test_get_permanent_id(self, test, expected):
         from kallithea.lib.utils import _get_permanent_id
         extracted = _get_permanent_id(test)
         assert extracted == expected, 'url:%s, got:`%s` expected: `%s`' % (test, base._test, expected)
 
     @base.parametrize('test,expected', [
       ("", ""),
       ("/", "/"),
       ("/_ID", '/_ID'),
       ("ID", "ID"),
       ("_ID", 'NAME'),
       ("_ID/", 'NAME/'),
       ("_ID/1/2", 'NAME/1/2'),
       ("_IDa", '_IDa'),
     ])
     def test_fix_repo_id_name(self, test, expected):
         repo = db.Repository.get_by_repo_name(base.HG_REPO)
         test = test.replace('ID', str(repo.repo_id))
         expected = expected.replace('NAME', repo.repo_name).replace('ID', str(repo.repo_id))
         from kallithea.lib.utils import fix_repo_id_name
         replaced = fix_repo_id_name(test)
         assert replaced == expected, 'url:%s, got:`%s` expected: `%s`' % (test, replaced, expected)
 
     @base.parametrize('canonical,test,expected', [
         ('http://www.example.org/', '/abc/xyz', 'http://www.example.org/abc/xyz'),
         ('http://www.example.org', '/abc/xyz', 'http://www.example.org/abc/xyz'),
         ('http://www.example.org', '/abc/xyz/', 'http://www.example.org/abc/xyz/'),
         ('http://www.example.org', 'abc/xyz/', 'http://www.example.org/abc/xyz/'),
         ('http://www.example.org', 'about', 'http://www.example.org/about-page'),
         ('http://www.example.org/repos/', 'abc/xyz/', 'http://www.example.org/repos/abc/xyz/'),
         ('http://www.example.org/kallithea/repos/', 'abc/xyz/', 'http://www.example.org/kallithea/repos/abc/xyz/'),
     ])
     def test_canonical_url(self, canonical, test, expected):
         # setup url(), used by canonical_url
-        import routes
-        from tg import request
-
         m = routes.Mapper()
         m.connect('about', '/about-page')
         url = routes.URLGenerator(m, {'HTTP_HOST': 'http_host.example.org'})
 
         config_mock = {
             'canonical_url': canonical,
         }
 
         with test_context(self.app):
             request.environ['routes.url'] = url
             with mock.patch('kallithea.CONFIG', config_mock):
                 assert webutils.canonical_url(test) == expected
 
     @base.parametrize('canonical,expected', [
         ('http://www.example.org', 'www.example.org'),
         ('http://www.example.org/repos/', 'www.example.org'),
         ('http://www.example.org/kallithea/repos/', 'www.example.org'),
     ])
     def test_canonical_hostname(self, canonical, expected):
-        import routes
-        from tg import request
-
         # setup url(), used by canonical_hostname
         m = routes.Mapper()
         url = routes.URLGenerator(m, {'HTTP_HOST': 'http_host.example.org'})
 
         config_mock = {
             'canonical_url': canonical,
         }
 
         with test_context(self.app):
             request.environ['routes.url'] = url
             with mock.patch('kallithea.CONFIG', config_mock):
                 assert webutils.canonical_hostname() == expected

kallithea/tests/performance/test_vcs.py

@@ -1,37 +1,37 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import pytest
 
+from kallithea.lib.graphmod import graph_data
 from kallithea.model import db
 from kallithea.tests import base
 
 
 @pytest.mark.skipif("'TEST_PERFORMANCE' not in os.environ", reason="skipping performance tests, set TEST_PERFORMANCE in environment if desired")
 class TestVCSPerformance(base.TestController):
 
     def graphmod(self, repo):
         """ Simple test for running the graph_data function for profiling/testing performance. """
-        from kallithea.lib.graphmod import graph_data
         dbr = db.Repository.get_by_repo_name(repo)
         scm_inst = dbr.scm_instance
         collection = scm_inst.get_changesets(start=0, end=None, branch_name=None)
         revs = [x.revision for x in collection]
         jsdata = graph_data(scm_inst, revs)
 
     def test_graphmod_hg(self, benchmark):
         benchmark(self.graphmod, base.HG_REPO)
 
     def test_graphmod_git(self, benchmark):
         benchmark(self.graphmod, base.GIT_REPO)

kallithea/tests/scripts/manual_test_concurrency.py

@@ -1,212 +1,212 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_concurrency
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test suite for making push/pull operations
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 
 """
 
 import logging
 import os
 import shutil
 import sys
 import tempfile
 from os.path import dirname
 from subprocess import PIPE, Popen
 
 from paste.deploy import appconfig
 from sqlalchemy import engine_from_config
 
 import kallithea.config.application
 from kallithea.lib.auth import get_crypt_password
 from kallithea.model import db, meta
 from kallithea.model.base import init_model
+from kallithea.model.repo import RepoModel
 from kallithea.tests.base import HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 
 
 rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
 conf = appconfig('config:development.ini', relative_to=rel_path)
 kallithea.config.application.make_app(conf.global_conf, **conf.local_conf)
 
 USER = TEST_USER_ADMIN_LOGIN
 PASS = TEST_USER_ADMIN_PASS
 HOST = 'server.local'
 METHOD = 'pull'
 DEBUG = True
 log = logging.getLogger(__name__)
 
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
 
     def execute(self, cmd, *args):
         """Runs command on the system with given ``args``.
         """
 
         command = cmd + ' ' + ' '.join(args)
         log.debug('Executing %s', command)
         if DEBUG:
             print(command)
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
         stdout, stderr = p.communicate()
         if DEBUG:
             print(stdout, stderr)
         return stdout, stderr
 
 
 def get_session():
     engine = engine_from_config(conf, 'sqlalchemy.')
     init_model(engine)
     sa = meta.Session
     return sa
 
 
 def create_test_user(force=True):
     print('creating test user')
     sa = get_session()
 
     user = sa.query(db.User).filter(db.User.username == USER).scalar()
 
     if force and user is not None:
         print('removing current user')
         for repo in sa.query(db.Repository).filter(db.Repository.user == user).all():
             sa.delete(repo)
         sa.delete(user)
         sa.commit()
 
     if user is None or force:
         print('creating new one')
         new_usr = db.User()
         new_usr.username = USER
         new_usr.password = get_crypt_password(PASS)
         new_usr.email = '[e-mail]'
         new_usr.name = 'test'
         new_usr.lastname = 'lasttestname'
         new_usr.active = True
         new_usr.admin = True
         sa.add(new_usr)
         sa.commit()
 
     print('done')
 
 
 def create_test_repo(force=True):
     print('creating test repo')
-    from kallithea.model.repo import RepoModel
     sa = get_session()
 
     user = sa.query(db.User).filter(db.User.username == USER).scalar()
     if user is None:
         raise Exception('user not found')
 
     repo = sa.query(db.Repository).filter(db.Repository.repo_name == HG_REPO).scalar()
 
     if repo is None:
         print('repo not found creating')
 
         form_data = {'repo_name': HG_REPO,
                      'repo_type': 'hg',
                      'private': False,
                      'clone_uri': ''}
         rm = RepoModel()
         rm.base_path = '/home/hg'
         rm.create(form_data, user)
 
     print('done')
 
 
 def set_anonymous_access(enable=True):
     sa = get_session()
     user = sa.query(db.User).filter(db.User.username == 'default').one()
     user.active = enable
     sa.add(user)
     sa.commit()
 
 
 def get_anonymous_access():
     sa = get_session()
     return sa.query(db.User).filter(db.User.username == 'default').one().active
 
 
 #==============================================================================
 # TESTS
 #==============================================================================
 def test_clone_with_credentials(no_errors=False, repo=HG_REPO, method=METHOD,
                                 backend='hg'):
     cwd = path = os.path.join(db.Ui.get_by_key('paths', '/').ui_value, repo)
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % os.path.join(path)
     except OSError:
         raise
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user': USER,
                    'pass': PASS,
                    'host': HOST,
                    'cloned_repo': repo, }
 
     dest = tempfile.mktemp(dir=path, prefix='dest-')
     if method == 'pull':
         stdout, stderr = Command(cwd).execute(backend, method, '--cwd', dest, clone_url)
     else:
         stdout, stderr = Command(cwd).execute(backend, method, clone_url, dest)
         if not no_errors:
             if backend == 'hg':
                 assert """adding file changes""" in stdout, 'no messages about cloning'
                 assert """abort""" not in stderr, 'got error from clone'
             elif backend == 'git':
                 assert """Cloning into""" in stdout, 'no messages about cloning'
 
 
 if __name__ == '__main__':
     try:
         create_test_user(force=False)
         import time
 
         try:
             METHOD = sys.argv[3]
         except IndexError:
             pass
 
         try:
             backend = sys.argv[4]
         except IndexError:
             backend = 'hg'
 
         if METHOD == 'pull':
             seq = next(tempfile._RandomNameSequence())
             test_clone_with_credentials(repo=sys.argv[1], method='clone',
                                         backend=backend)
         s = time.time()
         for i in range(1, int(sys.argv[2]) + 1):
             print('take', i)
             test_clone_with_credentials(repo=sys.argv[1], method=METHOD,
                                         backend=backend)
         print('time taken %.3f' % (time.time() - s))
     except Exception as e:
         sys.exit('stop on %s' % e)

kallithea/tests/vcs/test_repository.py

@@ -1,256 +1,256 b''
+import copy
 import datetime
 
 import pytest
 
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.tests.vcs import TEST_USER_CONFIG_FILE
 from kallithea.tests.vcs.base import _BackendTestMixin
 
 
 class RepositoryBaseTest(_BackendTestMixin):
     recreate_repo_per_test = False
 
     @classmethod
     def _get_commits(cls):
         return super(RepositoryBaseTest, cls)._get_commits()[:1]
 
     def test_get_config_value(self):
         assert self.repo.get_config_value('universal', 'foo', TEST_USER_CONFIG_FILE) == 'bar'
 
     def test_get_config_value_defaults_to_None(self):
         assert self.repo.get_config_value('universal', 'nonexist', TEST_USER_CONFIG_FILE) == None
 
     def test_get_user_name(self):
         assert self.repo.get_user_name(TEST_USER_CONFIG_FILE) == 'Foo Bar'
 
     def test_get_user_email(self):
         assert self.repo.get_user_email(TEST_USER_CONFIG_FILE) == '[e-mail]'
 
     def test_repo_equality(self):
         assert self.repo == self.repo
 
     def test_repo_equality_broken_object(self):
-        import copy
         _repo = copy.copy(self.repo)
         delattr(_repo, 'path')
         assert self.repo != _repo
 
     def test_repo_equality_other_object(self):
         class dummy(object):
             path = self.repo.path
         assert self.repo != dummy()
 
 
 class TestGitRepositoryBase(RepositoryBaseTest):
     backend_alias = 'git'
 
 
 class TestHgRepositoryBase(RepositoryBaseTest):
     backend_alias = 'hg'
 
 
 class RepositoryGetDiffTest(_BackendTestMixin):
 
     @classmethod
     def _get_commits(cls):
         commits = [
             {
                 'message': 'Initial commit',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 20),
                 'added': [
                     FileNode('foobar', content='foobar'),
                     FileNode('foobar2', content='foobar2'),
                 ],
             },
             {
                 'message': 'Changed foobar, added foobar3',
                 'author': 'Jane Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 21),
                 'added': [
                     FileNode('foobar3', content='foobar3'),
                 ],
                 'changed': [
                     FileNode('foobar', 'FOOBAR'),
                 ],
             },
             {
                 'message': 'Removed foobar, changed foobar3',
                 'author': 'Jane Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 22),
                 'changed': [
                     FileNode('foobar3', content='FOOBAR\nFOOBAR\nFOOBAR\n'),
                 ],
                 'removed': [FileNode('foobar')],
             },
             {
                 'message': 'Commit that contains glob pattern in filename',
                 'author': 'Jane Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 22),
                 'added': [
                     FileNode('README{', content='Strangely-named README file'),
                 ],
             },
         ]
         return commits
 
     def test_raise_for_wrong(self):
         with pytest.raises(ChangesetDoesNotExistError):
             self.repo.get_diff('a' * 40, 'b' * 40)
 
     def test_glob_patterns_in_filename_do_not_raise_exception(self):
         revs = self.repo.revisions
 
         diff = self.repo.get_diff(revs[2], revs[3], path='README{') # should not raise
 
 
 class TestGitRepositoryGetDiff(RepositoryGetDiffTest):
     backend_alias = 'git'
 
     def test_initial_commit_diff(self):
         initial_rev = self.repo.revisions[0]
         assert self.repo.get_diff(self.repo.EMPTY_CHANGESET, initial_rev) == br'''diff --git a/foobar b/foobar
 new file mode 100644
 index 0000000000000000000000000000000000000000..f6ea0495187600e7b2288c8ac19c5886383a4632
 --- /dev/null
 +++ b/foobar
 @@ -0,0 +1 @@
 +foobar
 \ No newline at end of file
 diff --git a/foobar2 b/foobar2
 new file mode 100644
 index 0000000000000000000000000000000000000000..e8c9d6b98e3dce993a464935e1a53f50b56a3783
 --- /dev/null
 +++ b/foobar2
 @@ -0,0 +1 @@
 +foobar2
 \ No newline at end of file
 '''
 
     def test_second_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[0], revs[1]) == br'''diff --git a/foobar b/foobar
 index f6ea0495187600e7b2288c8ac19c5886383a4632..389865bb681b358c9b102d79abd8d5f941e96551 100644
 --- a/foobar
 +++ b/foobar
 @@ -1 +1 @@
 -foobar
 \ No newline at end of file
 +FOOBAR
 \ No newline at end of file
 diff --git a/foobar3 b/foobar3
 new file mode 100644
 index 0000000000000000000000000000000000000000..c11c37d41d33fb47741cff93fa5f9d798c1535b0
 --- /dev/null
 +++ b/foobar3
 @@ -0,0 +1 @@
 +foobar3
 \ No newline at end of file
 '''
 
     def test_third_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[1], revs[2]) == br'''diff --git a/foobar b/foobar
 deleted file mode 100644
 index 389865bb681b358c9b102d79abd8d5f941e96551..0000000000000000000000000000000000000000
 --- a/foobar
 +++ /dev/null
 @@ -1 +0,0 @@
 -FOOBAR
 \ No newline at end of file
 diff --git a/foobar3 b/foobar3
 index c11c37d41d33fb47741cff93fa5f9d798c1535b0..f9324477362684ff692aaf5b9a81e01b9e9a671c 100644
 --- a/foobar3
 +++ b/foobar3
 @@ -1 +1,3 @@
 -foobar3
 \ No newline at end of file
 +FOOBAR
 +FOOBAR
 +FOOBAR
 '''
 
     def test_fourth_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[2], revs[3]) == br'''diff --git a/README{ b/README{
 new file mode 100644
 index 0000000000000000000000000000000000000000..cdc0c1b5d234feedb37bbac19cd1b6442061102d
 --- /dev/null
 +++ b/README{
 @@ -0,0 +1 @@
 +Strangely-named README file
 \ No newline at end of file
 '''
 
 
 class TestHgRepositoryGetDiff(RepositoryGetDiffTest):
     backend_alias = 'hg'
 
     def test_initial_commit_diff(self):
         initial_rev = self.repo.revisions[0]
         assert self.repo.get_diff(self.repo.EMPTY_CHANGESET, initial_rev) == br'''diff --git a/foobar b/foobar
 new file mode 100644
 --- /dev/null
 +++ b/foobar
 @@ -0,0 +1,1 @@
 +foobar
 \ No newline at end of file
 diff --git a/foobar2 b/foobar2
 new file mode 100644
 --- /dev/null
 +++ b/foobar2
 @@ -0,0 +1,1 @@
 +foobar2
 \ No newline at end of file
 '''
 
     def test_second_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[0], revs[1]) == br'''diff --git a/foobar b/foobar
 --- a/foobar
 +++ b/foobar
 @@ -1,1 +1,1 @@
 -foobar
 \ No newline at end of file
 +FOOBAR
 \ No newline at end of file
 diff --git a/foobar3 b/foobar3
 new file mode 100644
 --- /dev/null
 +++ b/foobar3
 @@ -0,0 +1,1 @@
 +foobar3
 \ No newline at end of file
 '''
 
     def test_third_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[1], revs[2]) == br'''diff --git a/foobar b/foobar
 deleted file mode 100644
 --- a/foobar
 +++ /dev/null
 @@ -1,1 +0,0 @@
 -FOOBAR
 \ No newline at end of file
 diff --git a/foobar3 b/foobar3
 --- a/foobar3
 +++ b/foobar3
 @@ -1,1 +1,3 @@
 -foobar3
 \ No newline at end of file
 +FOOBAR
 +FOOBAR
 +FOOBAR
 '''
 
     def test_fourth_changeset_diff(self):
         revs = self.repo.revisions
         assert self.repo.get_diff(revs[2], revs[3]) == br'''diff --git a/README{ b/README{
 new file mode 100644
 --- /dev/null
 +++ b/README{
 @@ -0,0 +1,1 @@
 +Strangely-named README file
 \ No newline at end of file
 '''

kallithea/tests/vcs/test_workdirs.py

@@ -1,91 +1,90 b''
 import datetime
 
 import pytest
 
+from kallithea.lib.vcs.exceptions import BranchDoesNotExistError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.tests.vcs.base import _BackendTestMixin
 
 
 class WorkdirTestCaseMixin(_BackendTestMixin):
 
     @classmethod
     def _get_commits(cls):
         commits = [
             {
                 'message': 'Initial commit',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 20),
                 'added': [
                     FileNode('foobar', content='Foobar'),
                     FileNode('foobar2', content='Foobar II'),
                     FileNode('foo/bar/baz', content='baz here!'),
                 ],
             },
             {
                 'message': 'Changes...',
                 'author': 'Jane Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 21),
                 'added': [
                     FileNode('some/new.txt', content='news...'),
                 ],
                 'changed': [
                     FileNode('foobar', 'Foobar I'),
                 ],
                 'removed': [],
             },
         ]
         return commits
 
     def test_get_branch_for_default_branch(self):
         assert self.repo.workdir.get_branch() == self.repo.DEFAULT_BRANCH_NAME
 
     def test_get_branch_after_adding_one(self):
         self.imc.add(FileNode('docs/index.txt',
             content='Documentation\n'))
         self.imc.commit(
             message='New branch: foobar',
             author='joe',
             branch='foobar',
         )
         assert self.repo.workdir.get_branch() == self.default_branch
 
     def test_get_changeset(self):
         old_head = self.repo.get_changeset()
         self.imc.add(FileNode('docs/index.txt',
             content='Documentation\n'))
         head = self.imc.commit(
             message='New branch: foobar',
             author='joe',
             branch='foobar',
         )
         assert self.repo.workdir.get_branch() == self.default_branch
         self.repo.workdir.checkout_branch('foobar')
         assert self.repo.workdir.get_changeset() == head
 
         # Make sure that old head is still there after update to default branch
         self.repo.workdir.checkout_branch(self.default_branch)
         assert self.repo.workdir.get_changeset() == old_head
 
     def test_checkout_branch(self):
-        from kallithea.lib.vcs.exceptions import BranchDoesNotExistError
-
         # first, 'foobranch' does not exist.
         with pytest.raises(BranchDoesNotExistError):
             self.repo.workdir.checkout_branch(branch='foobranch')
         # create new branch 'foobranch'.
         self.imc.add(FileNode('file1', content='blah'))
         self.imc.commit(message='asd', author='john', branch='foobranch')
         # go back to the default branch
         self.repo.workdir.checkout_branch()
         assert self.repo.workdir.get_branch() == self.backend_class.DEFAULT_BRANCH_NAME
         # checkout 'foobranch'
         self.repo.workdir.checkout_branch('foobranch')
         assert self.repo.workdir.get_branch() == 'foobranch'
 
 
 class TestGitBranch(WorkdirTestCaseMixin):
     backend_alias = 'git'
 
 
 class TestHgBranch(WorkdirTestCaseMixin):
     backend_alias = 'hg'

setup.py

@@ -1,161 +1,161 b''
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
 import platform
+import re
 import sys
 
 import setuptools
 # monkey patch setuptools to use distutils owner/group functionality
 from setuptools.command import sdist
 
 
 if sys.version_info < (3, 6):
     raise Exception('Kallithea requires Python 3.6 or later')
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 def _get_meta_var(name, data, callback_handler=None):
-    import re
     matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
     if matches:
         s = eval(matches.groups()[0])
         if callable(callback_handler):
             return callback_handler(s)
         return s
 
 _meta = open(os.path.join(here, 'kallithea', '__init__.py'), 'r')
 _metadata = _meta.read()
 _meta.close()
 
 def callback(V):
     return '.'.join(map(str, V[:3])) + '.'.join(V[3:])
 __version__ = _get_meta_var('VERSION', _metadata, callback)
 __license__ = _get_meta_var('__license__', _metadata)
 __author__ = _get_meta_var('__author__', _metadata)
 __url__ = _get_meta_var('__url__', _metadata)
 # defines current platform
 __platform__ = platform.system()
 
 is_windows = __platform__ in ['Windows']
 
 requirements = [
     "alembic >= 1.0.10, < 1.5",
     "gearbox >= 0.1.0, < 1",
     "waitress >= 0.8.8, < 1.5",
     "WebOb >= 1.8, < 1.9",
     "backlash >= 0.1.2, < 1",
     "TurboGears2 >= 2.4, < 2.5",
     "tgext.routes >= 0.2.0, < 1",
     "Beaker >= 1.10.1, < 2",
     "WebHelpers2 >= 2.0, < 2.1",
     "FormEncode >= 1.3.1, < 1.4",
     "SQLAlchemy >= 1.2.9, < 1.4",
     "Mako >= 0.9.1, < 1.2",
     "Pygments >= 2.2.0, < 2.7",
     "Whoosh >= 2.7.1, < 2.8",
     "celery >= 4.3, < 4.5, != 4.4.4", # 4.4.4 is broken due to unexpressed dependency on 'future', see https://github.com/celery/celery/pull/6146
     "Babel >= 1.3, < 2.9",
     "python-dateutil >= 2.1.0, < 2.9",
     "Markdown >= 2.2.1, < 3.2",
     "docutils >= 0.11, < 0.17",
     "URLObject >= 2.3.4, < 2.5",
     "Routes >= 2.0, < 2.5",
     "dulwich >= 0.19.0, < 0.20",
     "mercurial >= 5.2, < 5.7",
     "decorator >= 4.2.1, < 4.5",
     "Paste >= 2.0.3, < 3.5",
     "bleach >= 3.0, < 3.1.4",
     "Click >= 7.0, < 8",
     "ipaddr >= 2.2.0, < 2.3",
     "paginate >= 0.5, < 0.6",
     "paginate_sqlalchemy >= 0.3.0, < 0.4",
     "bcrypt >= 3.1.0, < 3.2",
     "pip >= 20.0, < 999",
 ]
 
 dependency_links = [
 ]
 
 classifiers = [
     'Development Status :: 4 - Beta',
     'Environment :: Web Environment',
     'Framework :: Pylons',
     'Intended Audience :: Developers',
     'License :: OSI Approved :: GNU General Public License (GPL)',
     'Operating System :: OS Independent',
     'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
     'Topic :: Software Development :: Version Control',
 ]
 
 
 # additional files from project that goes somewhere in the filesystem
 # relative to sys.prefix
 data_files = []
 
 description = ('Kallithea is a fast and powerful management tool '
                'for Mercurial and Git with a built in push/pull server, '
                'full text search and code-review.')
 
 keywords = ' '.join([
     'kallithea', 'mercurial', 'git', 'code review',
     'repo groups', 'ldap', 'repository management', 'hgweb replacement',
     'hgwebdir', 'gitweb replacement', 'serving hgweb',
 ])
 
 # long description
 README_FILE = 'README.rst'
 try:
     long_description = open(README_FILE).read()
 except IOError as err:
     sys.stderr.write(
         "[WARNING] Cannot find file specified as long_description (%s): %s\n"
         % (README_FILE, err)
     )
     long_description = description
 
 
 sdist_org = sdist.sdist
 class sdist_new(sdist_org):
     def initialize_options(self):
         sdist_org.initialize_options(self)
         self.owner = self.group = 'root'
 sdist.sdist = sdist_new
 
 packages = setuptools.find_packages(exclude=['ez_setup'])
 
 setuptools.setup(
     name='Kallithea',
     version=__version__,
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author=__author__,
     author_email='[e-mail]',
     dependency_links=dependency_links,
     url=__url__,
     install_requires=requirements,
     classifiers=classifiers,
     data_files=data_files,
     packages=packages,
     include_package_data=True,
     message_extractors={'kallithea': [
             ('**.py', 'python', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None)]},
     zip_safe=False,
     entry_points="""
     [console_scripts]
     kallithea-api =    kallithea.bin.kallithea_api:main
     kallithea-gist =   kallithea.bin.kallithea_gist:main
     kallithea-cli =    kallithea.bin.kallithea_cli:cli
 
     [paste.app_factory]
     main = kallithea.config.application:make_app
     """,
 )