py3: remove usage of basestring

rhodecode/api/views/user_api.py

@@ -1,563 +1,564 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
+from pyramid import compat
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error, store_update)
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import AuthUser, PasswordGenerator
 from rhodecode.lib.exceptions import DefaultUserException
 from rhodecode.lib.utils2 import safe_int, str2bool
 from rhodecode.model.db import Session, User, Repository
 from rhodecode.model.user import UserModel
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import user_schema
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_user(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Returns the information associated with a username or userid.
 
     * If the ``userid`` is not set, this command returns the information
       for the ``userid`` calling the method.
 
     .. note::
 
        Normal users may only run this command against their ``userid``. For
        full privileges you must run this command using an |authtoken| with
        admin rights.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which data will be returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         {
           "error": null,
           "id": <id>,
           "result": {
             "active": true,
             "admin": false,
             "api_keys": [ list of keys ],
             "auth_tokens": [ list of tokens with details ],
             "email": "[e-mail]",
             "emails": [
               "[e-mail]"
             ],
             "extern_name": "rhodecode",
             "extern_type": "rhodecode",
             "firstname": "username",
             "ip_addresses": [],
             "language": null,
             "last_login": "Timestamp",
             "last_activity": "Timestamp",
             "lastname": "surnae",
             "permissions": <deprecated>,
             "permissions_summary": {
               "global": [
                 "hg.inherit_default_perms.true",
                 "usergroup.read",
                 "hg.repogroup.create.false",
                 "hg.create.none",
                 "hg.password_reset.enabled",
                 "hg.extern_activate.manual",
                 "hg.create.write_on_repogroup.false",
                 "hg.usergroup.create.false",
                 "group.none",
                 "repository.none",
                 "hg.register.none",
                 "hg.fork.repository"
               ],
               "repositories": { "username/example": "repository.write"},
               "repositories_groups": { "user-group/repo": "group.none" },
               "user_groups": { "user_group_name": "usergroup.read" }
             }
             "user_id": 32,
             "username": "username"
           }
         }
     """
 
     if not has_superadmin_permission(apiuser):
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     data = user.get_api_data(include_secrets=True)
     permissions = AuthUser(user_id=user.user_id).permissions
     data['permissions'] = permissions  # TODO(marcink): should be deprecated
     data['permissions_summary'] = permissions
     return data
 
 
 @jsonrpc_method()
 def get_users(request, apiuser):
     """
     Lists all users in the |RCE| user database.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [<user_object>, ...]
         error:  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     result = []
     users_list = User.query().order_by(User.username) \
         .filter(User.username != User.DEFAULT_USER) \
         .all()
     for user in users_list:
         result.append(user.get_api_data(include_secrets=True))
     return result
 
 
 @jsonrpc_method()
 def create_user(request, apiuser, username, email, password=Optional(''),
                 firstname=Optional(''), lastname=Optional(''),
                 active=Optional(True), admin=Optional(False),
                 extern_name=Optional('rhodecode'),
                 extern_type=Optional('rhodecode'),
                 force_password_change=Optional(False),
                 create_personal_repo_group=Optional(None)):
     """
     Creates a new user and returns the new user object.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param username: Set the new username.
     :type username: str or int
     :param email: Set the user email address.
     :type email: str
     :param password: Set the new user password.
     :type password: Optional(str)
     :param firstname: Set the new user firstname.
     :type firstname: Optional(str)
     :param lastname: Set the new user surname.
     :type lastname: Optional(str)
     :param active: Set the user as active.
     :type active: Optional(``True`` | ``False``)
     :param admin: Give the new user admin rights.
     :type admin: Optional(``True`` | ``False``)
     :param extern_name: Set the authentication plugin name.
         Using LDAP this is filled with LDAP UID.
     :type extern_name: Optional(str)
     :param extern_type: Set the new user authentication plugin.
     :type extern_type: Optional(str)
     :param force_password_change: Force the new user to change password
         on next login.
     :type force_password_change: Optional(``True`` | ``False``)
     :param create_personal_repo_group: Create personal repo group for this user
     :type create_personal_repo_group: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
             "msg" : "created new user `<username>`",
             "user": <user_obj>
         }
         error:  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "user `<username>` already exist"
         or
         "email `<email>` already exist"
         or
         "failed to create user `<username>`"
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     if UserModel().get_by_username(username):
         raise JSONRPCError("user `%s` already exist" % (username,))
 
     if UserModel().get_by_email(email, case_insensitive=True):
         raise JSONRPCError("email `%s` already exist" % (email,))
 
     # generate random password if we actually given the
     # extern_name and it's not rhodecode
     if (not isinstance(extern_name, Optional) and
                 Optional.extract(extern_name) != 'rhodecode'):
         # generate temporary password if user is external
         password = PasswordGenerator().gen_password(length=16)
     create_repo_group = Optional.extract(create_personal_repo_group)
-    if isinstance(create_repo_group, basestring):
+    if isinstance(create_repo_group, compat.string_types):
         create_repo_group = str2bool(create_repo_group)
 
     username = Optional.extract(username)
     password = Optional.extract(password)
     email = Optional.extract(email)
     first_name = Optional.extract(firstname)
     last_name = Optional.extract(lastname)
     active = Optional.extract(active)
     admin = Optional.extract(admin)
     extern_type = Optional.extract(extern_type)
     extern_name = Optional.extract(extern_name)
 
     schema = user_schema.UserSchema().bind(
         # user caller
         user=apiuser)
     try:
         schema_data = schema.deserialize(dict(
             username=username,
             email=email,
             password=password,
             first_name=first_name,
             last_name=last_name,
             active=active,
             admin=admin,
             extern_type=extern_type,
             extern_name=extern_name,
             ))
     except validation_schema.Invalid as err:
         raise JSONRPCValidationError(colander_exc=err)
 
     try:
         user = UserModel().create_or_update(
             username=schema_data['username'],
             password=schema_data['password'],
             email=schema_data['email'],
             firstname=schema_data['first_name'],
             lastname=schema_data['last_name'],
             active=schema_data['active'],
             admin=schema_data['admin'],
             extern_type=schema_data['extern_type'],
             extern_name=schema_data['extern_name'],
             force_password_change=Optional.extract(force_password_change),
             create_repo_group=create_repo_group
         )
         Session().flush()
         creation_data = user.get_api_data()
         audit_logger.store_api(
             'user.create', action_data={'data': creation_data},
             user=apiuser)
 
         Session().commit()
         return {
             'msg': 'created new user `%s`' % username,
             'user': user.get_api_data(include_secrets=True)
         }
     except Exception:
         log.exception('Error occurred during creation of user')
         raise JSONRPCError('failed to create user `%s`' % (username,))
 
 
 @jsonrpc_method()
 def update_user(request, apiuser, userid, username=Optional(None),
                 email=Optional(None), password=Optional(None),
                 firstname=Optional(None), lastname=Optional(None),
                 active=Optional(None), admin=Optional(None),
                 extern_type=Optional(None), extern_name=Optional(None), ):
     """
     Updates the details for the specified user, if that user exists.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Set the ``userid`` to update.
     :type userid: str or int
     :param username: Set the new username.
     :type username: str or int
     :param email: Set the new email.
     :type email: str
     :param password: Set the new password.
     :type password: Optional(str)
     :param firstname: Set the new first name.
     :type firstname: Optional(str)
     :param lastname: Set the new surname.
     :type lastname: Optional(str)
     :param active: Set the new user as active.
     :type active: Optional(``True`` | ``False``)
     :param admin: Give the user admin rights.
     :type admin: Optional(``True`` | ``False``)
     :param extern_name: Set the authentication plugin user name.
         Using LDAP this is filled with LDAP UID.
     :type extern_name: Optional(str)
     :param extern_type: Set the authentication plugin type.
     :type extern_type: Optional(str)
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
             "msg" : "updated user ID:<userid> <username>",
             "user": <user_object>,
         }
         error:  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "failed to update user `<username>`"
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     user = get_user_or_error(userid)
     old_data = user.get_api_data()
     # only non optional arguments will be stored in updates
     updates = {}
 
     try:
 
         store_update(updates, username, 'username')
         store_update(updates, password, 'password')
         store_update(updates, email, 'email')
         store_update(updates, firstname, 'name')
         store_update(updates, lastname, 'lastname')
         store_update(updates, active, 'active')
         store_update(updates, admin, 'admin')
         store_update(updates, extern_name, 'extern_name')
         store_update(updates, extern_type, 'extern_type')
 
         user = UserModel().update_user(user, **updates)
         audit_logger.store_api(
             'user.edit', action_data={'old_data': old_data},
             user=apiuser)
         Session().commit()
         return {
             'msg': 'updated user ID:%s %s' % (user.user_id, user.username),
             'user': user.get_api_data(include_secrets=True)
         }
     except DefaultUserException:
         log.exception("Default user edit exception")
         raise JSONRPCError('editing default user is forbidden')
     except Exception:
         log.exception("Error occurred during update of user")
         raise JSONRPCError('failed to update user `%s`' % (userid,))
 
 
 @jsonrpc_method()
 def delete_user(request, apiuser, userid):
     """
     Deletes the specified user from the |RCE| user database.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     .. important::
 
        Ensure all open pull requests and open code review
        requests to this user are close.
 
        Also ensure all repositories, or repository groups owned by this
        user are reassigned before deletion.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param userid: Set the user to delete.
     :type userid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
             "msg" : "deleted user ID:<userid> <username>",
             "user": null
         }
         error:  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "failed to delete user ID:<userid> <username>"
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     user = get_user_or_error(userid)
     old_data = user.get_api_data()
     try:
         UserModel().delete(userid)
         audit_logger.store_api(
             'user.delete', action_data={'old_data': old_data},
             user=apiuser)
 
         Session().commit()
         return {
             'msg': 'deleted user ID:%s %s' % (user.user_id, user.username),
             'user': None
         }
     except Exception:
         log.exception("Error occurred during deleting of user")
         raise JSONRPCError(
             'failed to delete user ID:%s %s' % (user.user_id, user.username))
 
 
 @jsonrpc_method()
 def get_user_locks(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays all repositories locked by the specified user.
 
     * If this command is run by a non-admin user, it returns
       a list of |repos| locked by that user.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid whose list of locked |repos| will be
         displayed.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
             [repo_object, repo_object,...]
         }
         error :  null
     """
 
     include_secrets = False
     if not has_superadmin_permission(apiuser):
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
     else:
         include_secrets = True
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
     user = get_user_or_error(userid)
 
     ret = []
 
     # show all locks
     for r in Repository.getAll():
         _user_id, _time, _reason = r.locked
         if _user_id and _time:
             _api_data = r.get_api_data(include_secrets=include_secrets)
             # if we use user filter just show the locks for this user
             if safe_int(_user_id) == user.user_id:
                 ret.append(_api_data)
 
     return ret
 
 
 @jsonrpc_method()
 def get_user_audit_logs(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Fetches all action logs made by the specified user.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid whose list of locked |repos| will be
         displayed.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
             [action, action,...]
         }
         error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
     user = get_user_or_error(userid)
 
     ret = []
 
     # show all user actions
     for entry in UserModel().get_user_log(user, filter_term=None):
         ret.append(entry)
     return ret

rhodecode/apps/_base/__init__.py

@@ -1,686 +1,687 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 import operator
 
+from pyramid import compat
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest
 
 from rhodecode.lib import helpers as h, diffs
 from rhodecode.lib.utils2 import (
     StrictAttributeDict, safe_int, datetime_to_time, safe_unicode)
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
 from rhodecode.model import repo_group
 from rhodecode.model import user_group
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 ADMIN_PREFIX = '/_admin'
 STATIC_FILE_PREFIX = '/_static'
 
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     'group_name': r'.*?[^/]',
     'repo_group_name': r'.*?[^/]',
     # repo names can have a slash in them, but they must not end with a slash
     'repo_name': r'.*?[^/]',
     # file path eats up everything at the end
     'f_path': r'.*',
     # reference types
     'source_ref_type': '(branch|book|tag|rev|\%\(source_ref_type\)s)',
     'target_ref_type': '(branch|book|tag|rev|\%\(target_ref_type\)s)',
 }
 
 
 def add_route_with_slash(config,name, pattern, **kw):
     config.add_route(name, pattern, **kw)
     if not pattern.endswith('/'):
         config.add_route(name + '_slash', pattern + '/', **kw)
 
 
 def add_route_requirements(route_path, requirements=None):
     """
     Adds regex requirements to pyramid routes using a mapping dict
     e.g::
         add_route_requirements('{repo_name}/settings')
     """
     requirements = requirements or URL_NAME_REQUIREMENTS
     for key, regex in requirements.items():
         route_path = route_path.replace('{%s}' % key, '{%s:%s}' % (key, regex))
     return route_path
 
 
 def get_format_ref_id(repo):
     """Returns a `repo` specific reference formatter function"""
     if h.is_svn(repo):
         return _format_ref_id_svn
     else:
         return _format_ref_id
 
 
 def _format_ref_id(name, raw_id):
     """Default formatting of a given reference `name`"""
     return name
 
 
 def _format_ref_id_svn(name, raw_id):
     """Special way of formatting a reference for Subversion including path"""
     return '%s@%s' % (name, raw_id)
 
 
 class TemplateArgs(StrictAttributeDict):
     pass
 
 
 class BaseAppView(object):
 
     def __init__(self, context, request):
         self.request = request
         self.context = context
         self.session = request.session
         if not hasattr(request, 'user'):
             # NOTE(marcink): edge case, we ended up in matched route
             # but probably of web-app context, e.g API CALL/VCS CALL
             if hasattr(request, 'vcs_call') or hasattr(request, 'rpc_method'):
                 log.warning('Unable to process request `%s` in this scope', request)
                 raise HTTPBadRequest()
 
         self._rhodecode_user = request.user  # auth user
         self._rhodecode_db_user = self._rhodecode_user.get_instance()
         self._maybe_needs_password_change(
             request.matched_route.name, self._rhodecode_db_user)
 
     def _maybe_needs_password_change(self, view_name, user_obj):
         log.debug('Checking if user %s needs password change on view %s',
                   user_obj, view_name)
         skip_user_views = [
             'logout', 'login',
             'my_account_password', 'my_account_password_update'
         ]
 
         if not user_obj:
             return
 
         if user_obj.username == User.DEFAULT_USER:
             return
 
         now = time.time()
         should_change = user_obj.user_data.get('force_password_change')
         change_after = safe_int(should_change) or 0
         if should_change and now > change_after:
             log.debug('User %s requires password change', user_obj)
             h.flash('You are required to change your password', 'warning',
                     ignore_duplicate=True)
 
             if view_name not in skip_user_views:
                 raise HTTPFound(
                     self.request.route_path('my_account_password'))
 
     def _log_creation_exception(self, e, repo_name):
         _ = self.request.translate
         reason = None
         if len(e.args) == 2:
             reason = e.args[1]
 
         if reason == 'INVALID_CERTIFICATE':
             log.exception(
                 'Exception creating a repository: invalid certificate')
             msg = (_('Error creating repository %s: invalid certificate')
                    % repo_name)
         else:
             log.exception("Exception creating a repository")
             msg = (_('Error creating repository %s')
                    % repo_name)
         return msg
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         c = TemplateArgs()
         c.auth_user = self.request.user
         # TODO(marcink): migrate the usage of c.rhodecode_user to c.auth_user
         c.rhodecode_user = self.request.user
 
         if include_app_defaults:
             from rhodecode.lib.base import attach_context_attributes
             attach_context_attributes(c, self.request, self.request.user.user_id)
 
         return c
 
     def _get_template_context(self, tmpl_args, **kwargs):
 
         local_tmpl_args = {
             'defaults': {},
             'errors': {},
             'c': tmpl_args
         }
         local_tmpl_args.update(kwargs)
         return local_tmpl_args
 
     def load_default_context(self):
         """
         example:
 
         def load_default_context(self):
             c = self._get_local_tmpl_context()
             c.custom_var = 'foobar'
 
             return c
         """
         raise NotImplementedError('Needs implementation in view class')
 
 
 class RepoAppView(BaseAppView):
 
     def __init__(self, context, request):
         super(RepoAppView, self).__init__(context, request)
         self.db_repo = request.db_repo
         self.db_repo_name = self.db_repo.repo_name
         self.db_repo_pull_requests = ScmModel().get_pull_requests(self.db_repo)
 
     def _handle_missing_requirements(self, error):
         log.error(
             'Requirements are missing for repository %s: %s',
             self.db_repo_name, safe_unicode(error))
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super(RepoAppView, self)._get_local_tmpl_context(
             include_app_defaults=include_app_defaults)
 
         # register common vars for this type of view
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
         self.path_filter = PathFilter(None)
 
         c.repository_requirements_missing = {}
         try:
             self.rhodecode_vcs_repo = self.db_repo.scm_instance()
             if self.rhodecode_vcs_repo:
                 path_perms = self.rhodecode_vcs_repo.get_path_permissions(
                     c.auth_user.username)
                 self.path_filter = PathFilter(path_perms)
         except RepositoryRequirementError as e:
             c.repository_requirements_missing = {'error': str(e)}
             self._handle_missing_requirements(e)
             self.rhodecode_vcs_repo = None
 
         c.path_filter = self.path_filter  # used by atom_feed_entry.mako
 
         if self.rhodecode_vcs_repo is None:
             # unable to fetch this repo as vcs instance, report back to user
             h.flash(_(
                 "The repository `%(repo_name)s` cannot be loaded in filesystem. "
                 "Please check if it exist, or is not damaged.") %
                     {'repo_name': c.repo_name},
                     category='error', ignore_duplicate=True)
             if c.repository_requirements_missing:
                 route = self.request.matched_route.name
                 if route.startswith(('edit_repo', 'repo_summary')):
                     # allow summary and edit repo on missing requirements
                     return c
 
                 raise HTTPFound(
                     h.route_path('repo_summary', repo_name=self.db_repo_name))
 
             else:  # redirect if we don't show missing requirements
                 raise HTTPFound(h.route_path('home'))
 
         c.has_origin_repo_read_perm = False
         if self.db_repo.fork:
             c.has_origin_repo_read_perm = h.HasRepoPermissionAny(
                 'repository.write', 'repository.read', 'repository.admin')(
                 self.db_repo.fork.repo_name, 'summary fork link')
 
         return c
 
     def _get_f_path_unchecked(self, matchdict, default=None):
         """
         Should only be used by redirects, everything else should call _get_f_path
         """
         f_path = matchdict.get('f_path')
         if f_path:
             # fix for multiple initial slashes that causes errors for GIT
             return f_path.lstrip('/')
 
         return default
 
     def _get_f_path(self, matchdict, default=None):
         f_path_match = self._get_f_path_unchecked(matchdict, default)
         return self.path_filter.assert_path_permissions(f_path_match)
 
     def _get_general_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
 
 class PathFilter(object):
 
     # Expects and instance of BasePathPermissionChecker or None
     def __init__(self, permission_checker):
         self.permission_checker = permission_checker
 
     def assert_path_permissions(self, path):
         if path and self.permission_checker and not self.permission_checker.has_access(path):
             raise HTTPForbidden()
         return path
 
     def filter_patchset(self, patchset):
         if not self.permission_checker or not patchset:
             return patchset, False
         had_filtered = False
         filtered_patchset = []
         for patch in patchset:
             filename = patch.get('filename', None)
             if not filename or self.permission_checker.has_access(filename):
                 filtered_patchset.append(patch)
             else:
                 had_filtered = True
         if had_filtered:
             if isinstance(patchset, diffs.LimitedDiffContainer):
                 filtered_patchset = diffs.LimitedDiffContainer(patchset.diff_limit, patchset.cur_diff_size, filtered_patchset)
             return filtered_patchset, True
         else:
             return patchset, False
 
     def render_patchset_filtered(self, diffset, patchset, source_ref=None, target_ref=None):
         filtered_patchset, has_hidden_changes = self.filter_patchset(patchset)
         result = diffset.render_patchset(
             filtered_patchset, source_ref=source_ref, target_ref=target_ref)
         result.has_hidden_changes = has_hidden_changes
         return result
 
     def get_raw_patch(self, diff_processor):
         if self.permission_checker is None:
             return diff_processor.as_raw()
         elif self.permission_checker.has_full_access:
             return diff_processor.as_raw()
         else:
             return '# Repository has user-specific filters, raw patch generation is disabled.'
 
     @property
     def is_enabled(self):
         return self.permission_checker is not None
 
 
 class RepoGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super(RepoGroupAppView, self).__init__(context, request)
         self.db_repo_group = request.db_repo_group
         self.db_repo_group_name = self.db_repo_group.group_name
 
     def _revoke_perms_on_yourself(self, form_result):
         _updates = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                           form_result['perm_updates'])
         _additions = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                             form_result['perm_additions'])
         _deletions = filter(lambda u: self._rhodecode_user.user_id == int(u[0]),
                             form_result['perm_deletions'])
         admin_perm = 'group.admin'
         if _updates and _updates[0][1] != admin_perm or \
            _additions and _additions[0][1] != admin_perm or \
            _deletions and _deletions[0][1] != admin_perm:
             return True
         return False
 
 
 class UserGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super(UserGroupAppView, self).__init__(context, request)
         self.db_user_group = request.db_user_group
         self.db_user_group_name = self.db_user_group.users_group_name
 
 
 class UserAppView(BaseAppView):
     def __init__(self, context, request):
         super(UserAppView, self).__init__(context, request)
         self.db_user = request.db_user
         self.db_user_id = self.db_user.user_id
 
         _ = self.request.translate
         if not request.db_user_supports_default:
             if self.db_user.username == User.DEFAULT_USER:
                 h.flash(_("Editing user `{}` is disabled.".format(
                     User.DEFAULT_USER)), category='warning')
                 raise HTTPFound(h.route_path('users'))
 
 
 class DataGridAppView(object):
     """
     Common class to have re-usable grid rendering components
     """
 
     def _extract_ordering(self, request, column_map=None):
         column_map = column_map or {}
         column_index = safe_int(request.GET.get('order[0][column]'))
         order_dir = request.GET.get(
             'order[0][dir]', 'desc')
         order_by = request.GET.get(
             'columns[%s][data][sort]' % column_index, 'name_raw')
 
         # translate datatable to DB columns
         order_by = column_map.get(order_by) or order_by
 
         search_q = request.GET.get('search[value]')
         return search_q, order_by, order_dir
 
     def _extract_chunk(self, request):
         start = safe_int(request.GET.get('start'), 0)
         length = safe_int(request.GET.get('length'), 25)
         draw = safe_int(request.GET.get('draw'))
         return draw, start, length
 
     def _get_order_col(self, order_by, model):
-        if isinstance(order_by, basestring):
+        if isinstance(order_by, compat.string_types):
             try:
                 return operator.attrgetter(order_by)(model)
             except AttributeError:
                 return None
         else:
             return order_by
 
 
 class BaseReferencesView(RepoAppView):
     """
     Base for reference view for branches, tags and bookmarks.
     """
     def load_default_context(self):
         c = self._get_local_tmpl_context()
 
 
         return c
 
     def load_refs_context(self, ref_items, partials_template):
         _render = self.request.get_partial_renderer(partials_template)
         pre_load = ["author", "date", "message"]
 
         is_svn = h.is_svn(self.rhodecode_vcs_repo)
         is_hg = h.is_hg(self.rhodecode_vcs_repo)
 
         format_ref_id = get_format_ref_id(self.rhodecode_vcs_repo)
 
         closed_refs = {}
         if is_hg:
             closed_refs = self.rhodecode_vcs_repo.branches_closed
 
         data = []
         for ref_name, commit_id in ref_items:
             commit = self.rhodecode_vcs_repo.get_commit(
                 commit_id=commit_id, pre_load=pre_load)
             closed = ref_name in closed_refs
 
             # TODO: johbo: Unify generation of reference links
             use_commit_id = '/' in ref_name or is_svn
 
             if use_commit_id:
                 files_url = h.route_path(
                     'repo_files',
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else '',
                     commit_id=commit_id)
 
             else:
                 files_url = h.route_path(
                     'repo_files',
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else '',
                     commit_id=ref_name,
                     _query=dict(at=ref_name))
 
             data.append({
                 "name": _render('name', ref_name, files_url, closed),
                 "name_raw": ref_name,
                 "date": _render('date', commit.date),
                 "date_raw": datetime_to_time(commit.date),
                 "author": _render('author', commit.author),
                 "commit": _render(
                     'commit', commit.message, commit.raw_id, commit.idx),
                 "commit_raw": commit.idx,
                 "compare": _render(
                     'compare', format_ref_id(ref_name, commit.raw_id)),
             })
 
         return data
 
 
 class RepoRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         repo_name = info['match']['repo_name']
         repo_model = repo.RepoModel()
 
         by_name_match = repo_model.get_by_repo_name(repo_name, cache=False)
 
         def redirect_if_creating(route_info, db_repo):
             skip_views = ['edit_repo_advanced_delete']
             route = route_info['route']
             # we should skip delete view so we can actually "remove" repositories
             # if they get stuck in creating state.
             if route.name in skip_views:
                 return
 
             if db_repo.repo_state in [repo.Repository.STATE_PENDING]:
                 repo_creating_url = request.route_path(
                     'repo_creating', repo_name=db_repo.repo_name)
                 raise HTTPFound(repo_creating_url)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo = by_name_match
             redirect_if_creating(info, by_name_match)
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             request.db_repo = by_id_match
             redirect_if_creating(info, by_id_match)
             return True
 
         return False
 
 
 class RepoForbidArchivedRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_forbid_archived = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         _ = request.translate
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             '%s checking if archived flag for repo for %s',
             self.__class__.__name__, rhodecode_db_repo.repo_name)
 
         if rhodecode_db_repo.archived:
             log.warning('Current view is not supported for archived repo:%s',
                         rhodecode_db_repo.repo_name)
 
             h.flash(
                 h.literal(_('Action not supported for archived repository.')),
                 category='warning')
             summary_url = request.route_path(
                 'repo_summary', repo_name=rhodecode_db_repo.repo_name)
             raise HTTPFound(summary_url)
         return True
 
 
 class RepoTypeRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val or ['hg', 'git', 'svn']
 
     def text(self):
         return 'repo_accepted_type = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             '%s checking repo type for %s in %s',
             self.__class__.__name__, rhodecode_db_repo.repo_type, self.val)
 
         if rhodecode_db_repo.repo_type in self.val:
             return True
         else:
             log.warning('Current view is not supported for repo type:%s',
                         rhodecode_db_repo.repo_type)
             return False
 
 
 class RepoGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'repo_group_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         repo_group_name = info['match']['repo_group_name']
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name, cache=False)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo_group = by_name_match
             return True
 
         return False
 
 
 class UserGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'user_group_route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         user_group_id = info['match']['user_group_id']
         user_group_model = user_group.UserGroup()
         by_id_match = user_group_model.get(user_group_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user_group = by_id_match
             return True
 
         return False
 
 
 class UserRoutePredicateBase(object):
     supports_default = None
 
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         raise NotImplementedError()
 
     def __call__(self, info, request):
         if hasattr(request, 'vcs_call'):
             # skip vcs calls
             return
 
         user_id = info['match']['user_id']
         user_model = user.User()
         by_id_match = user_model.get(user_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user = by_id_match
             request.db_user_supports_default = self.supports_default
             return True
 
         return False
 
 
 class UserRoutePredicate(UserRoutePredicateBase):
     supports_default = False
 
     def text(self):
         return 'user_route = %s' % self.val
 
     phash = text
 
 
 class UserRouteWithDefaultPredicate(UserRoutePredicateBase):
     supports_default = True
 
     def text(self):
         return 'user_with_default_route = %s' % self.val
 
     phash = text
 
 
 def includeme(config):
     config.add_route_predicate(
         'repo_route', RepoRoutePredicate)
     config.add_route_predicate(
         'repo_accepted_types', RepoTypeRoutePredicate)
     config.add_route_predicate(
         'repo_forbid_when_archived', RepoForbidArchivedRoutePredicate)
     config.add_route_predicate(
         'repo_group_route', RepoGroupRoutePredicate)
     config.add_route_predicate(
         'user_group_route', UserGroupRoutePredicate)
     config.add_route_predicate(
         'user_route_with_default', UserRouteWithDefaultPredicate)
     config.add_route_predicate(
         'user_route', UserRoutePredicate)

rhodecode/apps/svn_support/__init__.py

@@ -1,90 +1,90 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-
+import os
 import logging
-import os
 import shlex
+from pyramid import compat
 
 # Do not use `from rhodecode import events` here, it will be overridden by the
 # events module in this package due to pythons import mechanism.
 from rhodecode.events import RepoGroupEvent
 from rhodecode.subscribers import AsyncSubprocessSubscriber
 from rhodecode.config.middleware import (
     _bool_setting, _string_setting, _int_setting)
 
 from .events import ModDavSvnConfigChange
 from .subscribers import generate_config_subscriber
 from . import config_keys
 
 
 log = logging.getLogger(__name__)
 
 
 def includeme(config):
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     if settings[config_keys.generate_config]:
         # Add subscriber to generate the Apache mod dav svn configuration on
         # repository group events.
         config.add_subscriber(generate_config_subscriber, RepoGroupEvent)
 
         # If a reload command is set add a subscriber to execute it on
         # configuration changes.
         reload_cmd = shlex.split(settings[config_keys.reload_command])
         if reload_cmd:
             reload_timeout = settings[config_keys.reload_timeout] or None
             reload_subscriber = AsyncSubprocessSubscriber(
                 cmd=reload_cmd, timeout=reload_timeout)
             config.add_subscriber(reload_subscriber, ModDavSvnConfigChange)
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
     _bool_setting(settings, config_keys.generate_config, 'false')
     _bool_setting(settings, config_keys.list_parent_path, 'true')
     _int_setting(settings, config_keys.reload_timeout, 10)
     _string_setting(settings, config_keys.config_file_path, '', lower=False)
     _string_setting(settings, config_keys.location_root, '/', lower=False)
     _string_setting(settings, config_keys.reload_command, '', lower=False)
     _string_setting(settings, config_keys.template, '', lower=False)
 
     # Convert negative timeout values to zero.
     if settings[config_keys.reload_timeout] < 0:
         settings[config_keys.reload_timeout] = 0
 
     # Append path separator to location root.
     settings[config_keys.location_root] = _append_path_sep(
         settings[config_keys.location_root])
 
     # Validate settings.
     if settings[config_keys.generate_config]:
         assert len(settings[config_keys.config_file_path]) > 0
 
 
 def _append_path_sep(path):
     """
     Append the path separator if missing.
     """
-    if isinstance(path, basestring) and not path.endswith(os.path.sep):
+    if isinstance(path, compat.string_types) and not path.endswith(os.path.sep):
         path += os.path.sep
     return path

rhodecode/lib/celerylib/tasks.py

@@ -1,339 +1,340 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
+from pyramid import compat
 from pyramid_mailer.mailer import Mailer
 from pyramid_mailer.message import Message
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask
 from rhodecode.lib.hooks_base import log_create_repository
 from rhodecode.lib.utils2 import safe_int, str2bool
 from rhodecode.model.db import Session, IntegrityError, Repository, User, true
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
 
     subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
 
     if recipients:
-        if isinstance(recipients, basestring):
+        if isinstance(recipients, compat.string_types):
             recipients = recipients.split(',')
     else:
         # if recipients are not defined we send to email_config + all admins
         admins = []
         for u in User.query().filter(User.admin == true()).all():
             if u.email:
                 admins.append(u.email)
         recipients = []
         config_email = email_config.get('email_to')
         if config_email:
             recipients += [config_email]
         recipients += admins
 
     # translate our LEGACY config into the one that pyramid_mailer supports
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
         username=email_config.get('smtp_username'),
         password=email_config.get('smtp_password'),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
 
         # SSL key file
         # keyfile='',
 
         # SSL certificate file
         # certfile='',
 
         # Location of maildir
         # queue_path='',
 
         default_sender=email_config.get('app_email_from', 'RhodeCode'),
 
         debug=str2bool(email_config.get('smtp_debug')),
         # /usr/sbin/sendmail	Sendmail executable
         # sendmail_app='',
 
         # {sendmail_app} -t -i -f {sender}	Template for sendmail execution
         # sendmail_template='',
     )
 
     try:
         mailer = Mailer(**email_conf)
 
         message = Message(subject=subject,
                           sender=email_conf['default_sender'],
                           recipients=recipients,
                           body=body, html=html_body)
         mailer.send_immediately(message)
 
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_rev = form_data['repo_landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     try:
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         repo.update_commit_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_rev = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
         repo.update_commit_cache(config=config)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True)
 def repo_maintenance(repoid):
     from rhodecode.lib import repo_maintenance as repo_maintenance_lib
     log = get_logger(repo_maintenance)
     repo = Repository.get_by_id_or_repo_name(repoid)
     if repo:
         maintenance = repo_maintenance_lib.RepoMaintenance()
         tasks = maintenance.get_tasks_for_repo(repo)
         log.debug('Executing %s tasks on repo `%s`', tasks, repoid)
         executed_types = maintenance.execute(repo)
         log.debug('Got execution results %s', executed_types)
     else:
         log.debug('Repo `%s` not found or without a clone_url', repoid)
 
 
 @async_task(ignore_result=True)
 def check_for_update():
     from rhodecode.model.update import UpdateModel
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
         latest = data['versions'][0]
         UpdateModel().store_version(latest['version'])
     except Exception:
         pass
 
 
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('Got args: %r and kwargs %r', args, kwargs)
     return time.time()

rhodecode/lib/codeblocks.py

@@ -1,775 +1,776 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
 from itertools import groupby
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from pygments.lexers.special import TextLexer, Token
 from pygments.lexers import get_lexer_by_name
+from pyramid import compat
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict, safe_unicode
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer, DEL_FILENODE, BIN_FILENODE
 
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
     log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
               lexer, filenode, org_lexer)
     tokens = tokenize_string(filenode.content, lexer)
     lines = split_token_stream(tokens)
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
 
     if isinstance(lexer, TextLexer):
         lexed = [(Token.Text, content)]
     else:
         lexed = lex(content, lexer)
 
     for token_type, token_text in lexed:
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     buffer = []
     for token_class, token_text in tokens:
         parts = token_text.split('\n')
         for part in parts[:-1]:
             buffer.append((token_class, part))
             yield buffer
             buffer = []
 
         buffer.append((token_class, parts[-1]))
 
     if buffer:
         yield buffer
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
             result.append(u'<span class="%s">' % token_class)
         else:
             result.append(u'<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
                 result.append(u'<%s>' % op_tag)
 
             escaped_text = html_escape(token_text)
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
                 result.append(u'</%s>' % op_tag)
 
         result.append(u'</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
     for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
         ops = []
         for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
     similarity = difflib.SequenceMatcher(None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
     if similarity < 0.6: # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
     token_sequence_matcher = difflib.SequenceMatcher(None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
         # check the differences by token block types first to give a more
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, 'del', token_text))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, 'ins', token_text))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
                 for char in token_text:
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
                 for char in token_text:
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
                 token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
                 token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11  # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], 'del', c))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
                             nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 def diffset_node_getter(commit):
     def get_node(fname):
         try:
             return commit.get_node(fname)
         except NodeDoesNotExistError:
             return None
 
     return get_node
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL'  # highlights using original file, slow
     HL_FAST = 'FAST'  # highlights using just the line, fast but not correct
                       # in the case of multiline code
     HL_NONE = 'NONE'  # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_repo_name=None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  # files over this size will use fast highlighting
                  max_file_size_limit=150 * 1024,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {}
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.target_repo_name = target_repo_name or repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             target_repo_name=self.target_repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = StrictAttributeDict(dict(
                 source_ref=diffset.source_ref,
                 target_ref=diffset.target_ref,
                 repo_name=diffset.repo_name,
                 source_repo_name=diffset.source_repo_name,
                 target_repo_name=diffset.target_repo_name,
             ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r', patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             node_hl_mode = self.HL_NONE if patch['chunks'] == [] else None
             hl_mode = node_hl_mode or self.highlight_mode
 
             if hl_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif hl_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
         raw_id_uid = ''
         if self.source_nodes.get(source_filename):
             raw_id_uid = self.source_nodes[source_filename].commit.raw_id
 
         if not raw_id_uid and self.target_nodes.get(target_filename):
             # in case this is a new file we only have it in target
             raw_id_uid = self.target_nodes[target_filename].commit.raw_id
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': patch['is_limited_diff'],
             'hunks': [],
             'hunk_ops': None,
             'diffset': self,
             'raw_id': raw_id_uid,
         })
 
         file_chunks = patch['chunks'][1:]
         for hunk in file_chunks:
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunks.append(hunkbit)
 
         # Simulate hunk on OPS type line which doesn't really contain any diff
         # this allows commenting on those
         if not file_chunks:
             actions = []
             for op_id, op_text in filediff.patch['stats']['ops'].items():
                 if op_id == DEL_FILENODE:
                     actions.append(u'file was removed')
                 elif op_id == BIN_FILENODE:
                     actions.append(u'binary diff hidden')
                 else:
                     actions.append(safe_unicode(op_text))
             action_line = u'NO CONTENT: ' + \
                           u', '.join(actions) or u'UNDEFINED_ACTION'
 
             hunk_ops = {'source_length': 0, 'source_start': 0,
                         'lines': [
                             {'new_lineno': 0, 'old_lineno': 1,
                              'action': 'unmod-no-hl', 'line': action_line}
                         ],
                         'section_header': u'', 'target_start': 1, 'target_length': 1}
 
             hunkbit = self.parse_hunk(hunk_ops, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunk_ops = hunkbit
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
             if line['action'] in ['unmod', 'unmod-no-hl']:
                 no_hl = line['action'] == 'unmod-no-hl'
                 result.lines.extend(
                     self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
             elif line['action'] == 'new-no-nl':
                 after.append(line)
 
         all_actions = [x['action'] for x in after] + [x['action'] for x in before]
         no_hl = {x for x in all_actions} == {'unmod-no-hl'}
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
         # NOTE(marcink): we must keep list() call here so we can cache the result...
         result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file,
                     no_hl=False):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
 
         before_newline = AttributeDict()
         after_newline = AttributeDict()
         if before_lines and before_lines[-1]['action'] == 'old-no-nl':
             before_newline_line = before_lines.pop(-1)
             before_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', before_newline_line['line'])]]))
 
         if after_lines and after_lines[-1]['action'] == 'new-no-nl':
             after_newline_line = after_lines.pop(-1)
             after_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', after_newline_line['line'])]]))
 
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
                     before_tokens = [('nonl', before['line'])]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'], line_number=before['old_lineno'],
                         input_file=source_file, no_hl=no_hl)
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
 
                 original.get_comment_args = (
                     source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
                     after_tokens = [('nonl', after['line'])]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         input_file=target_file, no_hl=no_hl)
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
 
                 modified.get_comment_args = (target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
             if not before_lines and before_newline:
                 original.content += before_newline.content
                 before_newline = None
             if not after_lines and after_newline:
                 modified.content += after_newline.content
                 after_newline = None
 
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_line_tokens(self, line_text, line_number, input_file=None, no_hl=False):
         filenode = None
         filename = None
 
-        if isinstance(input_file, basestring):
+        if isinstance(input_file, compat.string_types):
             filename = input_file
         elif isinstance(input_file, FileNode):
             filenode = input_file
             filename = input_file.unicode_path
 
         hl_mode = self.HL_NONE if no_hl else self.highlight_mode
         if hl_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
             file_size_allowed = input_file.size < self.max_file_size_limit
             if line_number and file_size_allowed:
                 return self.get_tokenized_filenode_line(
                     input_file, line_number, lexer)
 
         if hl_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None):
 
         if filenode not in self.highlighted_filenodes:
             tokenized_lines = filenode_as_lines_tokens(filenode, lexer)
             self.highlighted_filenodes[filenode] = tokenized_lines
         return self.highlighted_filenodes[filenode][line_number - 1]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'unmod-no-hl': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.get_comment_args)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/lib/colander_utils.py

@@ -1,30 +1,32 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+from pyramid import compat
+
 
 def strip_whitespace(value):
     """
     Removes leading/trailing whitespace, newlines, and tabs from the value.
     Implements the `colander.interface.Preparer` interface.
     """
-    if isinstance(value, basestring):
+    if isinstance(value, compat.string_types):
         return value.strip(' \t\n\r')
     else:
         return value

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

@@ -1,665 +1,666 b''
 """
    Schema module providing common schema operations.
 """
 import warnings
 
 from UserDict import DictMixin
 
 import sqlalchemy
 
 from sqlalchemy.schema import ForeignKeyConstraint
 from sqlalchemy.schema import UniqueConstraint
+from pyramid import compat
 
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07, SQLA_08
 from rhodecode.lib.dbmigrate.migrate.changeset import util
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (
     get_engine_visitor, run_single_visitor)
 
 
 __all__ = [
     'create_column',
     'drop_column',
     'alter_column',
     'rename_table',
     'rename_index',
     'ChangesetTable',
     'ChangesetColumn',
     'ChangesetIndex',
     'ChangesetDefaultClause',
     'ColumnDelta',
 ]
 
 def create_column(column, table=None, *p, **kw):
     """Create a column, given the table.
 
     API to :meth:`ChangesetColumn.create`.
     """
     if table is not None:
         return table.create_column(column, *p, **kw)
     return column.create(*p, **kw)
 
 
 def drop_column(column, table=None, *p, **kw):
     """Drop a column, given the table.
 
     API to :meth:`ChangesetColumn.drop`.
     """
     if table is not None:
         return table.drop_column(column, *p, **kw)
     return column.drop(*p, **kw)
 
 
 def rename_table(table, name, engine=None, **kw):
     """Rename a table.
 
     If Table instance is given, engine is not used.
 
     API to :meth:`ChangesetTable.rename`.
 
     :param table: Table to be renamed.
     :param name: New name for Table.
     :param engine: Engine instance.
     :type table: string or Table instance
     :type name: string
     :type engine: obj
     """
     table = _to_table(table, engine)
     table.rename(name, **kw)
 
 
 def rename_index(index, name, table=None, engine=None, **kw):
     """Rename an index.
 
     If Index instance is given,
     table and engine are not used.
 
     API to :meth:`ChangesetIndex.rename`.
 
     :param index: Index to be renamed.
     :param name: New name for index.
     :param table: Table to which Index is reffered.
     :param engine: Engine instance.
     :type index: string or Index instance
     :type name: string
     :type table: string or Table instance
     :type engine: obj
     """
     index = _to_index(index, table, engine)
     index.rename(name, **kw)
 
 
 def alter_column(*p, **k):
     """Alter a column.
 
     This is a helper function that creates a :class:`ColumnDelta` and
     runs it.
 
     :argument column:
       The name of the column to be altered or a
       :class:`ChangesetColumn` column representing it.
 
     :param table:
       A :class:`~sqlalchemy.schema.Table` or table name to
       for the table where the column will be changed.
 
     :param engine:
       The :class:`~sqlalchemy.engine.base.Engine` to use for table
       reflection and schema alterations.
 
     :returns: A :class:`ColumnDelta` instance representing the change.
 
 
     """
 
     if 'table' not in k and isinstance(p[0], sqlalchemy.Column):
         k['table'] = p[0].table
     if 'engine' not in k:
         k['engine'] = k['table'].bind
 
     # deprecation
     if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
         warnings.warn(
             "Passing a Column object to alter_column is deprecated."
             " Just pass in keyword parameters instead.",
             MigrateDeprecationWarning
             )
     engine = k['engine']
 
     # enough tests seem to break when metadata is always altered
     # that this crutch has to be left in until they can be sorted
     # out
     k['alter_metadata']=True
 
     delta = ColumnDelta(*p, **k)
 
     visitorcallable = get_engine_visitor(engine, 'schemachanger')
     engine._run_visitor(visitorcallable, delta)
 
     return delta
 
 
 def _to_table(table, engine=None):
     """Return if instance of Table, else construct new with metadata"""
     if isinstance(table, sqlalchemy.Table):
         return table
 
     # Given: table name, maybe an engine
     meta = sqlalchemy.MetaData()
     if engine is not None:
         meta.bind = engine
     return sqlalchemy.Table(table, meta)
 
 
 def _to_index(index, table=None, engine=None):
     """Return if instance of Index, else construct new with metadata"""
     if isinstance(index, sqlalchemy.Index):
         return index
 
     # Given: index name; table name required
     table = _to_table(table, engine)
     ret = sqlalchemy.Index(index)
     ret.table = table
     return ret
 
 
 class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
     """Extracts the differences between two columns/column-parameters
 
         May receive parameters arranged in several different ways:
 
         * **current_column, new_column, \*p, \*\*kw**
             Additional parameters can be specified to override column
             differences.
 
         * **current_column, \*p, \*\*kw**
             Additional parameters alter current_column. Table name is extracted
             from current_column object.
             Name is changed to current_column.name from current_name,
             if current_name is specified.
 
         * **current_col_name, \*p, \*\*kw**
             Table kw must specified.
 
         :param table: Table at which current Column should be bound to.\
         If table name is given, reflection will be used.
         :type table: string or Table instance
 
         :param metadata: A :class:`MetaData` instance to store
                          reflected table names
 
         :param engine: When reflecting tables, either engine or metadata must \
         be specified to acquire engine object.
         :type engine: :class:`Engine` instance
         :returns: :class:`ColumnDelta` instance provides interface for altered attributes to \
         `result_column` through :func:`dict` alike object.
 
         * :class:`ColumnDelta`.result_column is altered column with new attributes
 
         * :class:`ColumnDelta`.current_name is current name of column in db
 
 
     """
 
     # Column attributes that can be altered
     diff_keys = ('name', 'type', 'primary_key', 'nullable',
         'server_onupdate', 'server_default', 'autoincrement')
     diffs = dict()
     __visit_name__ = 'column'
 
     def __init__(self, *p, **kw):
         # 'alter_metadata' is not a public api. It exists purely
         # as a crutch until the tests that fail when 'alter_metadata'
         # behaviour always happens can be sorted out
         self.alter_metadata = kw.pop("alter_metadata", False)
 
         self.meta = kw.pop("metadata", None)
         self.engine = kw.pop("engine", None)
 
         # Things are initialized differently depending on how many column
         # parameters are given. Figure out how many and call the appropriate
         # method.
         if len(p) >= 1 and isinstance(p[0], sqlalchemy.Column):
             # At least one column specified
             if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
                 # Two columns specified
                 diffs = self.compare_2_columns(*p, **kw)
             else:
                 # Exactly one column specified
                 diffs = self.compare_1_column(*p, **kw)
         else:
             # Zero columns specified
-            if not len(p) or not isinstance(p[0], basestring):
+            if not len(p) or not isinstance(p[0], compat.string_types):
                 raise ValueError("First argument must be column name")
             diffs = self.compare_parameters(*p, **kw)
 
         self.apply_diffs(diffs)
 
     def __repr__(self):
         return '<ColumnDelta altermetadata=%r, %s>' % (
             self.alter_metadata,
             super(ColumnDelta, self).__repr__()
             )
 
     def __getitem__(self, key):
         if key not in self.keys():
             raise KeyError("No such diff key, available: %s" % self.diffs )
         return getattr(self.result_column, key)
 
     def __setitem__(self, key, value):
         if key not in self.keys():
             raise KeyError("No such diff key, available: %s" % self.diffs )
         setattr(self.result_column, key, value)
 
     def __delitem__(self, key):
         raise NotImplementedError
 
     def __len__(self):
         raise NotImplementedError
 
     def __iter__(self):
         raise NotImplementedError
 
     def keys(self):
         return self.diffs.keys()
 
     def compare_parameters(self, current_name, *p, **k):
         """Compares Column objects with reflection"""
         self.table = k.pop('table')
         self.result_column = self._table.c.get(current_name)
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_1_column(self, col, *p, **k):
         """Compares one Column object"""
         self.table = k.pop('table', None)
         if self.table is None:
             self.table = col.table
         self.result_column = col
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def compare_2_columns(self, old_col, new_col, *p, **k):
         """Compares two Column objects"""
         self.process_column(new_col)
         self.table = k.pop('table', None)
         # we cannot use bool() on table in SA06
         if self.table is None:
             self.table = old_col.table
         if self.table is None:
             new_col.table
         self.result_column = old_col
 
         # set differences
         # leave out some stuff for later comp
         for key in (set(self.diff_keys) - set(('type',))):
             val = getattr(new_col, key, None)
             if getattr(self.result_column, key, None) != val:
                 k.setdefault(key, val)
 
         # inspect types
         if not self.are_column_types_eq(self.result_column.type, new_col.type):
             k.setdefault('type', new_col.type)
 
         if len(p):
             k = self._extract_parameters(p, k, self.result_column)
         return k
 
     def apply_diffs(self, diffs):
         """Populate dict and column object with new values"""
         self.diffs = diffs
         for key in self.diff_keys:
             if key in diffs:
                 setattr(self.result_column, key, diffs[key])
 
         self.process_column(self.result_column)
 
         # create an instance of class type if not yet
         if 'type' in diffs and callable(self.result_column.type):
             self.result_column.type = self.result_column.type()
 
         # add column to the table
         if self.table is not None and self.alter_metadata:
             self.result_column.add_to_table(self.table)
 
     def are_column_types_eq(self, old_type, new_type):
         """Compares two types to be equal"""
         ret = old_type.__class__ == new_type.__class__
 
         # String length is a special case
         if ret and isinstance(new_type, sqlalchemy.types.String):
             ret = (getattr(old_type, 'length', None) == \
                        getattr(new_type, 'length', None))
         return ret
 
     def _extract_parameters(self, p, k, column):
         """Extracts data from p and modifies diffs"""
         p = list(p)
         while len(p):
-            if isinstance(p[0], basestring):
+            if isinstance(p[0], compat.string_types):
                 k.setdefault('name', p.pop(0))
             elif isinstance(p[0], sqlalchemy.types.TypeEngine):
                 k.setdefault('type', p.pop(0))
             elif callable(p[0]):
                 p[0] = p[0]()
             else:
                 break
 
         if len(p):
             new_col = column.copy_fixed()
             new_col._init_items(*p)
             k = self.compare_2_columns(column, new_col, **k)
         return k
 
     def process_column(self, column):
         """Processes default values for column"""
         # XXX: this is a snippet from SA processing of positional parameters
         toinit = list()
 
         if column.server_default is not None:
             if isinstance(column.server_default, sqlalchemy.FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_default))
         if column.server_onupdate is not None:
             if isinstance(column.server_onupdate, FetchedValue):
                 toinit.append(column.server_default)
             else:
                 toinit.append(sqlalchemy.DefaultClause(column.server_onupdate,
                                             for_update=True))
         if toinit:
             column._init_items(*toinit)
 
     def _get_table(self):
         return getattr(self, '_table', None)
 
     def _set_table(self, table):
-        if isinstance(table, basestring):
+        if isinstance(table, compat.string_types):
             if self.alter_metadata:
                 if not self.meta:
                     raise ValueError("metadata must be specified for table"
                         " reflection when using alter_metadata")
                 meta = self.meta
                 if self.engine:
                     meta.bind = self.engine
             else:
                 if not self.engine and not self.meta:
                     raise ValueError("engine or metadata must be specified"
                         " to reflect tables")
                 if not self.engine:
                     self.engine = self.meta.bind
                 meta = sqlalchemy.MetaData(bind=self.engine)
             self._table = sqlalchemy.Table(table, meta, autoload=True)
         elif isinstance(table, sqlalchemy.Table):
             self._table = table
             if not self.alter_metadata:
                 self._table.meta = sqlalchemy.MetaData(bind=self._table.bind)
     def _get_result_column(self):
         return getattr(self, '_result_column', None)
 
     def _set_result_column(self, column):
         """Set Column to Table based on alter_metadata evaluation."""
         self.process_column(column)
         if not hasattr(self, 'current_name'):
             self.current_name = column.name
         if self.alter_metadata:
             self._result_column = column
         else:
             self._result_column = column.copy_fixed()
 
     table = property(_get_table, _set_table)
     result_column = property(_get_result_column, _set_result_column)
 
 
 class ChangesetTable(object):
     """Changeset extensions to SQLAlchemy tables."""
 
     def create_column(self, column, *p, **kw):
         """Creates a column.
 
         The column parameter may be a column definition or the name of
         a column in this table.
 
         API to :meth:`ChangesetColumn.create`
 
         :param column: Column to be created
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             column = getattr(self.c, str(column))
         column.create(table=self, *p, **kw)
 
     def drop_column(self, column, *p, **kw):
         """Drop a column, given its name or definition.
 
         API to :meth:`ChangesetColumn.drop`
 
         :param column: Column to be droped
         :type column: Column instance or string
         """
         if not isinstance(column, sqlalchemy.Column):
             # It's a column name
             try:
                 column = getattr(self.c, str(column))
             except AttributeError:
                 # That column isn't part of the table. We don't need
                 # its entire definition to drop the column, just its
                 # name, so create a dummy column with the same name.
                 column = sqlalchemy.Column(str(column), sqlalchemy.Integer())
         column.drop(table=self, *p, **kw)
 
     def rename(self, name, connection=None, **kwargs):
         """Rename this table.
 
         :param name: New name of the table.
         :type name: string
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         engine = self.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         run_single_visitor(engine, visitorcallable, self, connection, **kwargs)
 
         # Fix metadata registration
         self.name = name
         self.deregister()
         self._set_parent(self.metadata)
 
     def _meta_key(self):
         """Get the meta key for this table."""
         return sqlalchemy.schema._get_table_key(self.name, self.schema)
 
     def deregister(self):
         """Remove this table from its metadata"""
         if SQLA_07:
             self.metadata._remove_table(self.name, self.schema)
         else:
             key = self._meta_key()
             meta = self.metadata
             if key in meta.tables:
                 del meta.tables[key]
 
 
 class ChangesetColumn(object):
     """Changeset extensions to SQLAlchemy columns."""
 
     def alter(self, *p, **k):
         """Makes a call to :func:`alter_column` for the column this
         method is called on.
         """
         if 'table' not in k:
             k['table'] = self.table
         if 'engine' not in k:
             k['engine'] = k['table'].bind
         return alter_column(self, *p, **k)
 
     def create(self, table=None, index_name=None, unique_name=None,
                primary_key_name=None, populate_default=True, connection=None, **kwargs):
         """Create this column in the database.
 
         Assumes the given table exists. ``ALTER TABLE ADD COLUMN``,
         for most databases.
 
         :param table: Table instance to create on.
         :param index_name: Creates :class:`ChangesetIndex` on this column.
         :param unique_name: Creates :class:\
 `~migrate.changeset.constraint.UniqueConstraint` on this column.
         :param primary_key_name: Creates :class:\
 `~migrate.changeset.constraint.PrimaryKeyConstraint` on this column.
         :param populate_default: If True, created column will be \
 populated with defaults
         :param connection: reuse connection istead of creating new one.
         :type table: Table instance
         :type index_name: string
         :type unique_name: string
         :type primary_key_name: string
         :type populate_default: bool
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
 
         :returns: self
         """
         self.populate_default = populate_default
         self.index_name = index_name
         self.unique_name = unique_name
         self.primary_key_name = primary_key_name
         for cons in ('index_name', 'unique_name', 'primary_key_name'):
             self._check_sanity_constraints(cons)
 
         self.add_to_table(table)
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columngenerator')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
 
         # TODO: reuse existing connection
         if self.populate_default and self.default is not None:
             stmt = table.update().values({self: engine._execute_default(self.default)})
             engine.execute(stmt)
 
         return self
 
     def drop(self, table=None, connection=None, **kwargs):
         """Drop this column from the database, leaving its table intact.
 
         ``ALTER TABLE DROP COLUMN``, for most databases.
 
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         if table is not None:
             self.table = table
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columndropper')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         self.remove_from_table(self.table, unset_table=False)
         self.table = None
         return self
 
     def add_to_table(self, table):
         if table is not None  and self.table is None:
             if SQLA_07:
                 table.append_column(self)
             else:
                 self._set_parent(table)
 
     def _col_name_in_constraint(self,cons,name):
         return False
 
     def remove_from_table(self, table, unset_table=True):
         # TODO: remove primary keys, constraints, etc
         if unset_table:
             self.table = None
 
         to_drop = set()
         for index in table.indexes:
             columns = []
             for col in index.columns:
                 if col.name!=self.name:
                     columns.append(col)
             if columns:
                 index.columns = columns
                 if SQLA_08:
                     index.expressions = columns
             else:
                 to_drop.add(index)
         table.indexes = table.indexes - to_drop
 
         to_drop = set()
         for cons in table.constraints:
             # TODO: deal with other types of constraint
             if isinstance(cons,(ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
-                    if not isinstance(col_name,basestring):
+                    if not isinstance(col_name, compat.string_types):
                         col_name = col_name.name
                     if self.name==col_name:
                         to_drop.add(cons)
         table.constraints = table.constraints - to_drop
 
         if table.c.contains_column(self):
             if SQLA_07:
                 table._columns.remove(self)
             else:
                 table.c.remove(self)
 
     # TODO: this is fixed in 0.6
     def copy_fixed(self, **kw):
         """Create a copy of this ``Column``, with all attributes."""
         q = util.safe_quote(self)
         return sqlalchemy.Column(self.name, self.type, self.default,
             key=self.key,
             primary_key=self.primary_key,
             nullable=self.nullable,
             quote=q,
             index=self.index,
             unique=self.unique,
             onupdate=self.onupdate,
             autoincrement=self.autoincrement,
             server_default=self.server_default,
             server_onupdate=self.server_onupdate,
             *[c.copy(**kw) for c in self.constraints])
 
     def _check_sanity_constraints(self, name):
         """Check if constraints names are correct"""
         obj = getattr(self, name)
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
-        if not isinstance(obj, basestring) and obj is not None:
+        if not isinstance(obj, compat.string_types) and obj is not None:
             raise InvalidConstraintError(
             "%s argument for column must be constraint name" % name)
 
 
 class ChangesetIndex(object):
     """Changeset extensions to SQLAlchemy Indexes."""
 
     __visit_name__ = 'index'
 
     def rename(self, name, connection=None, **kwargs):
         """Change the name of an index.
 
         :param name: New name of the Index.
         :type name: string
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         engine = self.table.bind
         self.new_name = name
         visitorcallable = get_engine_visitor(engine, 'schemachanger')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         self.name = name
 
 
 class ChangesetDefaultClause(object):
     """Implements comparison between :class:`DefaultClause` instances"""
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             if self.arg == other.arg:
                 return True
 
     def __ne__(self, other):
         return not self.__eq__(other)

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

@@ -1,221 +1,222 b''
 """
    Database schema version management.
 """
 import sys
 import logging
 
 from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
     create_engine)
 from sqlalchemy.sql import and_
 from sqlalchemy import exc as sa_exceptions
 from sqlalchemy.sql import bindparam
+from pyramid import compat
 
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
 from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
 from rhodecode.lib.dbmigrate.migrate.versioning.version import VerNum
 
 
 log = logging.getLogger(__name__)
 
 
 class ControlledSchema(object):
     """A database under version control"""
 
     def __init__(self, engine, repository):
-        if isinstance(repository, basestring):
+        if isinstance(repository, compat.string_types):
             repository = Repository(repository)
         self.engine = engine
         self.repository = repository
         self.meta = MetaData(engine)
         self.load()
 
     def __eq__(self, other):
         """Compare two schemas by repositories and versions"""
         return (self.repository is other.repository \
             and self.version == other.version)
 
     def load(self):
         """Load controlled schema version info from DB"""
         tname = self.repository.version_table
         try:
             if not hasattr(self, 'table') or self.table is None:
                     self.table = Table(tname, self.meta, autoload=True)
 
             result = self.engine.execute(self.table.select(
                 self.table.c.repository_id == str(self.repository.id)))
 
             data = list(result)[0]
         except:
             cls, exc, tb = sys.exc_info()
             raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
 
         self.version = data['version']
         return data
 
     def drop(self):
         """
         Remove version control from a database.
         """
         if SQLA_07:
             try:
                 self.table.drop()
             except sa_exceptions.DatabaseError:
                 raise exceptions.DatabaseNotControlledError(str(self.table))
         else:
             try:
                 self.table.drop()
             except (sa_exceptions.SQLError):
                 raise exceptions.DatabaseNotControlledError(str(self.table))
 
     def changeset(self, version=None):
         """API to Changeset creation.
 
         Uses self.version for start version and engine.name
         to get database name.
         """
         database = self.engine.name
         start_ver = self.version
         changeset = self.repository.changeset(database, start_ver, version)
         return changeset
 
     def runchange(self, ver, change, step):
         startver = ver
         endver = ver + step
         # Current database version must be correct! Don't run if corrupt!
         if self.version != startver:
             raise exceptions.InvalidVersionError("%s is not %s" % \
                                                      (self.version, startver))
         # Run the change
         change.run(self.engine, step)
 
         # Update/refresh database version
         self.update_repository_table(startver, endver)
         self.load()
 
     def update_repository_table(self, startver, endver):
         """Update version_table with new information"""
         update = self.table.update(and_(self.table.c.version == int(startver),
              self.table.c.repository_id == str(self.repository.id)))
         self.engine.execute(update, version=int(endver))
 
     def upgrade(self, version=None):
         """
         Upgrade (or downgrade) to a specified version, or latest version.
         """
         changeset = self.changeset(version)
         for ver, change in changeset:
             self.runchange(ver, change, changeset.step)
 
     def update_db_from_model(self, model):
         """
         Modify the database to match the structure of the current Python model.
         """
         model = load_model(model)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, self.engine, excludeTables=[self.repository.version_table]
             )
         genmodel.ModelGenerator(diff,self.engine).runB2A()
 
         self.update_repository_table(self.version, int(self.repository.latest))
 
         self.load()
 
     @classmethod
     def create(cls, engine, repository, version=None):
         """
         Declare a database to be under a repository's version control.
 
         :raises: :exc:`DatabaseAlreadyControlledError`
         :returns: :class:`ControlledSchema`
         """
         # Confirm that the version # is valid: positive, integer,
         # exists in repos
-        if isinstance(repository, basestring):
+        if isinstance(repository, compat.string_types):
             repository = Repository(repository)
         version = cls._validate_version(repository, version)
         table = cls._create_table_version(engine, repository, version)
         # TODO: history table
         # Load repository information and return
         return cls(engine, repository)
 
     @classmethod
     def _validate_version(cls, repository, version):
         """
         Ensures this is a valid version number for this repository.
 
         :raises: :exc:`InvalidVersionError` if invalid
         :return: valid version number
         """
         if version is None:
             version = 0
         try:
             version = VerNum(version) # raises valueerror
             if version < 0 or version > repository.latest:
                 raise ValueError()
         except ValueError:
             raise exceptions.InvalidVersionError(version)
         return version
 
     @classmethod
     def _create_table_version(cls, engine, repository, version):
         """
         Creates the versioning table in a database.
 
         :raises: :exc:`DatabaseAlreadyControlledError`
         """
         # Create tables
         tname = repository.version_table
         meta = MetaData(engine)
 
         table = Table(
             tname, meta,
             Column('repository_id', String(250), primary_key=True),
             Column('repository_path', Text),
             Column('version', Integer), )
 
         # there can be multiple repositories/schemas in the same db
         if not table.exists():
             table.create()
 
         # test for existing repository_id
         s = table.select(table.c.repository_id == bindparam("repository_id"))
         result = engine.execute(s, repository_id=repository.id)
         if result.fetchone():
             raise exceptions.DatabaseAlreadyControlledError
 
         # Insert data
         engine.execute(table.insert().values(
                            repository_id=repository.id,
                            repository_path=repository.path,
                            version=int(version)))
         return table
 
     @classmethod
     def compare_model_to_db(cls, engine, model, repository):
         """
         Compare the current model against the current database.
         """
-        if isinstance(repository, basestring):
+        if isinstance(repository, compat.string_types):
             repository = Repository(repository)
         model = load_model(model)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, engine, excludeTables=[repository.version_table])
         return diff
 
     @classmethod
     def create_model(cls, engine, repository, declarative=False):
         """
         Dump the current database as a Python model.
         """
-        if isinstance(repository, basestring):
+        if isinstance(repository, compat.string_types):
             repository = Repository(repository)
 
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             MetaData(), engine, excludeTables=[repository.version_table]
             )
         return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

@@ -1,159 +1,160 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import shutil
 import warnings
 import logging
 import inspect
 from StringIO import StringIO
 
+from pyramid import compat
 from rhodecode.lib.dbmigrate import migrate
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.config import operations
 from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
 from rhodecode.lib.dbmigrate.migrate.versioning.script import base
 from rhodecode.lib.dbmigrate.migrate.versioning.util import import_path, load_model, with_engine
 from rhodecode.lib.dbmigrate.migrate.exceptions import MigrateDeprecationWarning, InvalidScriptError, ScriptError
 
 log = logging.getLogger(__name__)
 __all__ = ['PythonScript']
 
 
 class PythonScript(base.BaseScript):
     """Base for Python scripts"""
 
     @classmethod
     def create(cls, path, **opts):
         """Create an empty migration script at specified path
 
         :returns: :class:`PythonScript instance <migrate.versioning.script.py.PythonScript>`"""
         cls.require_notfound(path)
 
         src = Template(opts.pop('templates_path', None)).get_script(theme=opts.pop('templates_theme', None))
         shutil.copy(src, path)
 
         return cls(path)
 
     @classmethod
     def make_update_script_for_model(cls, engine, oldmodel,
                                      model, repository, **opts):
         """Create a migration script based on difference between two SA models.
 
         :param repository: path to migrate repository
         :param oldmodel: dotted.module.name:SAClass or SAClass object
         :param model: dotted.module.name:SAClass or SAClass object
         :param engine: SQLAlchemy engine
         :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>`
         :type oldmodel: string or Class
         :type model: string or Class
         :type engine: Engine instance
         :returns: Upgrade / Downgrade script
         :rtype: string
         """
 
-        if isinstance(repository, basestring):
+        if isinstance(repository, compat.string_types):
             # oh dear, an import cycle!
             from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
             repository = Repository(repository)
 
         oldmodel = load_model(oldmodel)
         model = load_model(model)
 
         # Compute differences.
         diff = schemadiff.getDiffOfModelAgainstModel(
             model,
             oldmodel,
             excludeTables=[repository.version_table])
         # TODO: diff can be False (there is no difference?)
         decls, upgradeCommands, downgradeCommands = \
             genmodel.ModelGenerator(diff,engine).genB2AMigration()
 
         # Store differences into file.
         src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))
         with open(src) as f:
             contents = f.read()
 
         # generate source
         search = 'def upgrade(migrate_engine):'
         contents = contents.replace(search, '\n\n'.join((decls, search)), 1)
         if upgradeCommands:
             contents = contents.replace('    pass', upgradeCommands, 1)
         if downgradeCommands:
             contents = contents.replace('    pass', downgradeCommands, 1)
         return contents
 
     @classmethod
     def verify_module(cls, path):
         """Ensure path is a valid script
 
         :param path: Script location
         :type path: string
         :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
         :returns: Python module
         """
         # Try to import and get the upgrade() func
         module = import_path(path)
         try:
             assert callable(module.upgrade)
         except Exception as e:
             raise InvalidScriptError(path + ': %s' % str(e))
         return module
 
     def preview_sql(self, url, step, **args):
         """Mocks SQLAlchemy Engine to store all executed calls in a string
         and runs :meth:`PythonScript.run <migrate.versioning.script.py.PythonScript.run>`
 
         :returns: SQL file
         """
         buf = StringIO()
         args['engine_arg_strategy'] = 'mock'
         args['engine_arg_executor'] = lambda s, p = '': buf.write(str(s) + p)
 
         @with_engine
         def go(url, step, **kw):
             engine = kw.pop('engine')
             self.run(engine, step)
             return buf.getvalue()
 
         return go(url, step, **args)
 
     def run(self, engine, step):
         """Core method of Script file.
         Exectues :func:`update` or :func:`downgrade` functions
 
         :param engine: SQLAlchemy Engine
         :param step: Operation to run
         :type engine: string
         :type step: int
         """
         if step > 0:
             op = 'upgrade'
         elif step < 0:
             op = 'downgrade'
         else:
             raise ScriptError("%d is not a valid step" % step)
 
         funcname = base.operations[op]
         script_func = self._func(funcname)
 
         # check for old way of using engine
         if not inspect.getargspec(script_func)[0]:
             raise TypeError("upgrade/downgrade functions must accept engine"
                 " parameter (since version 0.5.4)")
 
         script_func(engine)
 
     @property
     def module(self):
         """Calls :meth:`migrate.versioning.script.py.verify_module`
         and returns it.
         """
         if not hasattr(self, '_module'):
             self._module = self.verify_module(self.path)
         return self._module
 
     def _func(self, funcname):
         if not hasattr(self.module, funcname):
             msg = "Function '%s' is not defined in this script"
             raise ScriptError(msg % funcname)
         return getattr(self.module, funcname)

rhodecode/lib/dbmigrate/migrate/versioning/util/__init__.py

@@ -1,179 +1,181 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """.. currentmodule:: migrate.versioning.util"""
 
 import warnings
 import logging
 from decorator import decorator
 from pkg_resources import EntryPoint
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.pool import StaticPool
 
+from pyramid import compat
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning.util.keyedinstance import KeyedInstance
 from rhodecode.lib.dbmigrate.migrate.versioning.util.importpath import import_path
 
 
 log = logging.getLogger(__name__)
 
+
 def load_model(dotted_name):
     """Import module and use module-level variable".
 
     :param dotted_name: path to model in form of string: ``some.python.module:Class``
 
     .. versionchanged:: 0.5.4
 
     """
-    if isinstance(dotted_name, basestring):
+    if isinstance(dotted_name, compat.string_types):
         if ':' not in dotted_name:
             # backwards compatibility
             warnings.warn('model should be in form of module.model:User '
                 'and not module.model.User', exceptions.MigrateDeprecationWarning)
             dotted_name = ':'.join(dotted_name.rsplit('.', 1))
         return EntryPoint.parse('x=%s' % dotted_name).load(False)
     else:
         # Assume it's already loaded.
         return dotted_name
 
 def asbool(obj):
     """Do everything to use object as bool"""
-    if isinstance(obj, basestring):
+    if isinstance(obj, compat.string_types):
         obj = obj.strip().lower()
         if obj in ['true', 'yes', 'on', 'y', 't', '1']:
             return True
         elif obj in ['false', 'no', 'off', 'n', 'f', '0']:
             return False
         else:
             raise ValueError("String is not true/false: %r" % obj)
     if obj in (True, False):
         return bool(obj)
     else:
         raise ValueError("String is not true/false: %r" % obj)
 
 def guess_obj_type(obj):
     """Do everything to guess object type from string
 
     Tries to convert to `int`, `bool` and finally returns if not succeded.
 
     .. versionadded: 0.5.4
     """
 
     result = None
 
     try:
         result = int(obj)
     except:
         pass
 
     if result is None:
         try:
             result = asbool(obj)
         except:
             pass
 
     if result is not None:
         return result
     else:
         return obj
 
 @decorator
 def catch_known_errors(f, *a, **kw):
     """Decorator that catches known api errors
 
     .. versionadded: 0.5.4
     """
 
     try:
         return f(*a, **kw)
     except exceptions.PathFoundError as e:
         raise exceptions.KnownError("The path %s already exists" % e.args[0])
 
 def construct_engine(engine, **opts):
     """.. versionadded:: 0.5.4
 
     Constructs and returns SQLAlchemy engine.
 
     Currently, there are 2 ways to pass create_engine options to :mod:`migrate.versioning.api` functions:
 
     :param engine: connection string or a existing engine
     :param engine_dict: python dictionary of options to pass to `create_engine`
     :param engine_arg_*: keyword parameters to pass to `create_engine` (evaluated with :func:`migrate.versioning.util.guess_obj_type`)
     :type engine_dict: dict
     :type engine: string or Engine instance
     :type engine_arg_*: string
     :returns: SQLAlchemy Engine
 
     .. note::
 
         keyword parameters override ``engine_dict`` values.
 
     """
     if isinstance(engine, Engine):
         return engine
-    elif not isinstance(engine, basestring):
+    elif not isinstance(engine, compat.string_types):
         raise ValueError("you need to pass either an existing engine or a database uri")
 
     # get options for create_engine
     if opts.get('engine_dict') and isinstance(opts['engine_dict'], dict):
         kwargs = opts['engine_dict']
     else:
         kwargs = {}
 
     # DEPRECATED: handle echo the old way
     echo = asbool(opts.get('echo', False))
     if echo:
         warnings.warn('echo=True parameter is deprecated, pass '
             'engine_arg_echo=True or engine_dict={"echo": True}',
             exceptions.MigrateDeprecationWarning)
         kwargs['echo'] = echo
 
     # parse keyword arguments
     for key, value in opts.iteritems():
         if key.startswith('engine_arg_'):
             kwargs[key[11:]] = guess_obj_type(value)
 
     log.debug('Constructing engine')
     # TODO: return create_engine(engine, poolclass=StaticPool, **kwargs)
     # seems like 0.5.x branch does not work with engine.dispose and staticpool
     return create_engine(engine, **kwargs)
 
 @decorator
 def with_engine(f, *a, **kw):
     """Decorator for :mod:`migrate.versioning.api` functions
     to safely close resources after function usage.
 
     Passes engine parameters to :func:`construct_engine` and
     resulting parameter is available as kw['engine'].
 
     Engine is disposed after wrapped function is executed.
 
     .. versionadded: 0.6.0
     """
     url = a[0]
     engine = construct_engine(url, **kw)
 
     try:
         kw['engine'] = engine
         return f(*a, **kw)
     finally:
         if isinstance(engine, Engine) and engine is not url:
             log.debug('Disposing SQLAlchemy engine %s', engine)
             engine.dispose()
 
 
 class Memoize:
     """Memoize(fn) - an instance which acts like fn but memoizes its arguments
        Will only work on functions with non-mutable arguments
 
        ActiveState Code 52201
     """
     def __init__(self, fn):
         self.fn = fn
         self.memo = {}
 
     def __call__(self, *args):
         if args not in self.memo:
             self.memo[args] = self.fn(*args)
         return self.memo[args]

rhodecode/lib/dbmigrate/schema/db_1_2_0.py

@@ -1,1043 +1,1044 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import logging
 import datetime
 import traceback
 from datetime import date
 
 from sqlalchemy import *
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
+from pyramid import compat
 
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.utils.helpers import get_scm
 from rhodecode.lib.vcs.exceptions import VCSError
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from rhodecode.lib.auth import generate_auth_token
 from rhodecode.lib.utils2 import str2bool, safe_str, get_commit_safe, safe_unicode
 from rhodecode.lib.exceptions import UserGroupAssignedException
 from rhodecode.lib.ext_json import json
 
 from rhodecode.model.meta import Base, Session
 from rhodecode.lib.caching_query import FromCache
 
 
 log = logging.getLogger(__name__)
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 class ModelSerializer(json.JSONEncoder):
     """
     Simple Serializer for JSON,
 
     usage::
 
         to make object customized for serialization implement a __json__
         method that will return a dict for serialization into json
 
     example::
 
         class Task(object):
 
             def __init__(self, name, value):
                 self.name = name
                 self.value = value
 
             def __json__(self):
                 return dict(name=self.name,
                             value=self.value)
 
     """
 
     def default(self, obj):
 
         if hasattr(obj, '__json__'):
             return obj.__json__()
         else:
             return json.JSONEncoder.default(self, obj)
 
 class BaseModel(object):
     """Base Model for all classess
 
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return Session.query(cls)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def getAll(cls):
         return cls.query().all()
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session.delete(obj)
         Session.commit()
 
 
 class RhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'extend_existing':True})
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None)
     _app_settings_value = Column("app_settings_value", String(255), nullable=True, unique=None, default=None)
 
     def __init__(self, k='', v=''):
         self.app_settings_name = k
         self.app_settings_value = v
 
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == unicode
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         if v == 'ldap_active':
             v = str2bool(v)
         return v
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.app_settings_name, self.app_settings_value)
 
 
     @classmethod
     def get_by_name(cls, ldap_key):
         return cls.query()\
             .filter(cls.app_settings_name == ldap_key).scalar()
 
     @classmethod
     def get_app_settings(cls, cache=False):
 
         ret = cls.query()
 
         if cache:
             ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 
         if not ret:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings['rhodecode_' + each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_ldap_settings(cls, cache=False):
         ret = cls.query()\
                 .filter(cls.app_settings_name.startswith('ldap_')).all()
         fd = {}
         for row in ret:
             fd.update({row.app_settings_name:row.app_settings_value})
 
         return fd
 
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = (UniqueConstraint('ui_key'), {'extend_existing':True})
 
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     HOOK_PUSH = 'pretxnchangegroup.push_logger'
     HOOK_PULL = 'preoutgoing.pull_logger'
 
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(255), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(255), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(255), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.ui_key == key)
 
 
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_REPO_SIZE,
                                      cls.HOOK_PUSH, cls.HOOK_PULL]))
         return q.all()
 
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_REPO_SIZE,
                                       cls.HOOK_PUSH, cls.HOOK_PULL]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
 
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_by_key(key).scalar() or cls()
         new_ui.ui_section = 'hooks'
         new_ui.ui_active = True
         new_ui.ui_key = key
         new_ui.ui_value = val
 
         Session.add(new_ui)
         Session.commit()
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'extend_existing':True})
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(255), nullable=True, unique=None, default=None)
     password = Column("password", String(255), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=None)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("name", String(255), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(255), nullable=True, unique=None, default=None)
     email = Column("email", String(255), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     ldap_dn = Column("ldap_dn", String(255), nullable=True, unique=None, default=None)
     api_key = Column("api_key", String(255), nullable=True, unique=None, default=None)
 
     user_log = relationship('UserLog', cascade='all')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relationship('Repository')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
 
     group_member = relationship('UserGroupMember', cascade='all')
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
 
     @property
     def short_contact(self):
         return '%s %s' % (self.name, self.lastname)
 
     @property
     def is_admin(self):
         return self.admin
 
     def __repr__(self):
         try:
             return "<%s('id:%s:%s')>" % (self.__class__.__name__,
                                              self.user_id, self.username)
         except:
             return self.__class__.__name__
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False):
         if case_insensitive:
             return Session.query(cls).filter(cls.username.ilike(username)).scalar()
         else:
             return Session.query(cls).filter(cls.username == username).scalar()
 
     @classmethod
     def get_by_auth_token(cls, auth_token):
         return cls.query().filter(cls.api_key == auth_token).one()
 
     def update_lastlogin(self):
         """Update user lastlogin"""
 
         self.last_login = datetime.datetime.now()
         Session.add(self)
         Session.commit()
         log.debug('updated user %s lastlogin', self.username)
 
     @classmethod
     def create(cls, form_data):
         from rhodecode.lib.auth import get_crypt_password
 
         try:
             new_user = cls()
             for k, v in form_data.items():
                 if k == 'password':
                     v = get_crypt_password(v)
                 setattr(new_user, k, v)
 
             new_user.api_key = generate_auth_token(form_data['username'])
             Session.add(new_user)
             Session.commit()
             return new_user
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = {'extend_existing':True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None)
     action = Column("action", String(1200000), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     @property
     def action_as_day(self):
         return date(*self.action_date.timetuple()[:3])
 
     user = relationship('User')
     repository = relationship('Repository')
 
 
 class UserGroup(Base, BaseModel):
     __tablename__ = 'users_groups'
     __table_args__ = {'extend_existing':True}
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None)
     users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
 
     members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined")
 
     def __repr__(self):
         return '<userGroup(%s)>' % (self.users_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.users_group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.users_group_name == group_name)
         if cache:
             gr = gr.options(FromCache("sql_cache_short",
                                           "get_user_%s" % group_name))
         return gr.scalar()
 
     @classmethod
     def get(cls, users_group_id, cache=False):
         users_group = cls.query()
         if cache:
             users_group = users_group.options(FromCache("sql_cache_short",
                                     "get_users_group_%s" % users_group_id))
         return users_group.get(users_group_id)
 
     @classmethod
     def create(cls, form_data):
         try:
             new_user_group = cls()
             for k, v in form_data.items():
                 setattr(new_user_group, k, v)
 
             Session.add(new_user_group)
             Session.commit()
             return new_user_group
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
     @classmethod
     def update(cls, users_group_id, form_data):
 
         try:
             users_group = cls.get(users_group_id, cache=False)
 
             for k, v in form_data.items():
                 if k == 'users_group_members':
                     users_group.members = []
                     Session.flush()
                     members_list = []
                     if v:
-                        v = [v] if isinstance(v, basestring) else v
+                        v = [v] if isinstance(v, compat.string_types) else v
                         for u_id in set(v):
                             member = UserGroupMember(users_group_id, u_id)
                             members_list.append(member)
                     setattr(users_group, 'members', members_list)
                 setattr(users_group, k, v)
 
             Session.add(users_group)
             Session.commit()
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
     @classmethod
     def delete(cls, user_group_id):
         try:
 
             # check if this group is not assigned to repo
             assigned_groups = UserGroupRepoToPerm.query()\
                 .filter(UserGroupRepoToPerm.users_group_id ==
                         user_group_id).all()
 
             if assigned_groups:
                 raise UserGroupAssignedException(
                     'UserGroup assigned to %s' % assigned_groups)
 
             users_group = cls.get(user_group_id, cache=False)
             Session.delete(users_group)
             Session.commit()
         except:
             log.error(traceback.format_exc())
             Session.rollback()
             raise
 
 class UserGroupMember(Base, BaseModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = {'extend_existing':True}
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', lazy='joined')
     users_group = relationship('UserGroup')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
     @staticmethod
     def add_user_to_group(group, user):
         ugm = UserGroupMember()
         ugm.users_group = group
         ugm.user = user
         Session.add(ugm)
         Session.commit()
         return ugm
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'extend_existing':True},)
 
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(255), nullable=False, unique=True, default=None)
     clone_uri = Column("clone_uri", String(255), nullable=True, unique=False, default=None)
     repo_type = Column("repo_type", String(255), nullable=False, unique=False, default='hg')
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(10000), nullable=True, unique=None, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     fork_id = Column("fork_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=False, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None)
 
 
     user = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
 
     logs = relationship('UserLog', cascade='all')
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.repo_id, self.repo_name)
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_repo_name(cls, repo_name):
         q = Session.query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .options(joinedload(Repository.group))
         return q.one()
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               cls.url_sep())
         q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def just_name(self):
         return self.repo_name.split(Repository.url_sep())[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert(0, cur_gr)
         while 1:
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert(0, gr)
 
         return groups
 
     @property
     def groups_and_repo(self):
         return self.groups_with_parents, self.just_name
 
     @LazyProperty
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = Session.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               Repository.url_sep())
         q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(Repository.url_sep())
         return os.path.join(*p)
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return Repository.url_sep().join(path_prefix + [repo_name])
 
     @property
     def _config(self):
         """
         Returns db based config object.
         """
         from rhodecode.lib.utils import make_db_config
         return make_db_config(clear_session=False)
 
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
 
         :param cls:
         :param repo_name:
         """
         from rhodecode.lib.utils import is_valid_repo
 
         return is_valid_repo(repo_name, cls.base_path())
 
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_commit(self, rev):
         return get_commit_safe(self.scm_instance, rev)
 
     @property
     def tip(self):
         return self.get_commit('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance.last_change
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     @property
     def invalidate(self):
         return CacheInvalidation.invalidate(self.repo_name)
 
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
         CacheInvalidation.set_invalidate(self.repo_name)
 
     @LazyProperty
     def scm_instance(self):
         return self.__get_instance()
 
     @property
     def scm_instance_cached(self):
         return self.__get_instance()
 
     def __get_instance(self):
 
         repo_full_path = self.repo_full_path
 
         try:
             alias = get_scm(repo_full_path)[0]
             log.debug('Creating instance of %s repository', alias)
             backend = get_backend(alias)
         except VCSError:
             log.error(traceback.format_exc())
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
 
         if alias == 'hg':
 
             repo = backend(safe_str(repo_full_path), create=False,
                            config=self._config)
 
         else:
             repo = backend(repo_full_path, create=False)
 
         return repo
 
 
 class Group(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'),
                       CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},)
     __mapper_args__ = {'order_by':'group_name'}
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None)
 
     parent_group = relationship('Group', remote_side=group_id)
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__, self.group_id,
                                   self.group_name)
 
     @classmethod
     def url_sep(cls):
         return '/'
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache("sql_cache_short",
                                           "get_group_%s" % group_name))
         return gr.scalar()
 
     @property
     def parents(self):
         parents_recursion_limit = 5
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert(0, cur_gr)
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error('group nested more than %s',
                           parents_recursion_limit)
                 break
 
             groups.insert(0, gr)
         return groups
 
     @property
     def children(self):
         return Group.query().filter(Group.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(Group.url_sep())[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(Group.url_sep())
 
     @property
     def repositories(self):
         return Repository.query().filter(Repository.group == self)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return Group.url_sep().join(path_prefix + [group_name])
 
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = {'extend_existing':True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.permission_id, self.permission_name)
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
 class UserRepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'extend_existing':True})
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'extend_existing':True})
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
 
     @classmethod
     def has_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         return cls.query().filter(cls.user_id == user_id)\
             .filter(cls.permission == perm).scalar() is not None
 
     @classmethod
     def grant_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         new = cls()
         new.user_id = user_id
         new.permission = perm
         try:
             Session.add(new)
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm(cls, user_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         try:
             cls.query().filter(cls.user_id == user_id) \
                 .filter(cls.permission == perm).delete()
             Session.commit()
         except:
             Session.rollback()
 
 class UserGroupRepoToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (UniqueConstraint('repository_id', 'users_group_id', 'permission_id'), {'extend_existing':True})
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
     def __repr__(self):
         return '<userGroup:%s => %s >' % (self.users_group, self.repository)
 
 class UserGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = {'extend_existing':True}
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 
 
     @classmethod
     def has_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         return cls.query().filter(cls.users_group_id ==
                                          users_group_id)\
                                          .filter(cls.permission == perm)\
                                          .scalar() is not None
 
     @classmethod
     def grant_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         new = cls()
         new.users_group_id = users_group_id
         new.permission = perm
         try:
             Session.add(new)
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm(cls, users_group_id, perm):
         if not isinstance(perm, Permission):
             raise Exception('perm needs to be an instance of Permission class')
 
         try:
             cls.query().filter(cls.users_group_id == users_group_id) \
                 .filter(cls.permission == perm).delete()
             Session.commit()
         except:
             Session.rollback()
 
 
 class UserRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'group_to_perm'
     __table_args__ = (UniqueConstraint('group_id', 'permission_id'), {'extend_existing':True})
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'extend_existing':True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data
 
     repository = relationship('Repository', single_parent=True)
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                       UniqueConstraint('user_id', 'follows_user_id')
                       , {'extend_existing':True})
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by='Repository.repo_name')
 
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing':True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
 
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
 
     @classmethod
     def invalidate(cls, key):
         """
         Returns Invalidation object if this given key should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
 
         :param key:
         """
         return cls.query()\
                 .filter(CacheInvalidation.cache_key == key)\
                 .filter(CacheInvalidation.cache_active == False)\
                 .scalar()
 
     @classmethod
     def set_invalidate(cls, key):
         """
         Mark this Cache key for invalidation
 
         :param key:
         """
 
         log.debug('marking %s for invalidation', key)
         inv_obj = Session.query(cls)\
             .filter(cls.cache_key == key).scalar()
         if inv_obj:
             inv_obj.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             inv_obj = CacheInvalidation(key)
 
         try:
             Session.add(inv_obj)
             Session.commit()
         except Exception:
             log.error(traceback.format_exc())
             Session.rollback()
 
     @classmethod
     def set_valid(cls, key):
         """
         Mark this cache key as active and currently cached
 
         :param key:
         """
         inv_obj = Session.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == key).scalar()
         inv_obj.cache_active = True
         Session.add(inv_obj)
         Session.commit()
 
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing':True}
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)