libs: major refactor for python3

rhodecode/lib/base.py

@@ -1,610 +1,606 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
+import base64
 
 import markupsafe
 import ipaddress
 
+import paste.httpheaders
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
-from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes)
-from rhodecode.lib.utils2 import (
-    str2bool, safe_unicode, AttributeDict, safe_int, sha1, aslist, safe_str)
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.str_utils import ascii_bytes, safe_int, safe_str
+from rhodecode.lib.type_utils import aslist, str2bool
+from rhodecode.lib.hash_utils import sha1
 from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_str(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
-    _filters = lambda x: _filter_port(_filter_proxy(x))
+
+    def ip_filters(ip_):
+        return _filter_port(_filter_proxy(ip_))
 
     ip = environ.get(proxy_key)
     if ip:
-        return _filters(ip)
+        return ip_filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
-        return _filters(ip)
+        return ip_filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
-    return _filters(ip)
+    return ip_filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
-def get_access_path(environ):
-    path = environ.get('PATH_INFO')
-    org_req = environ.get('pylons.original_request')
-    if org_req:
-        path = org_req.environ.get('PATH_INFO')
-    return path
-
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False, check_branch_perms=False, detect_force_push=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     user = User.get_by_username(username)
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
     user_id = user.user_id
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     # NOTE(marcink): This should be also in sync with
     # rhodecode/apps/ssh_support/lib/backends/base.py:update_environment scm_data
     store = [x for x in ui_settings if x.key == '/']
     repo_store = ''
     if store:
         repo_store = store[0].value
 
     scm_data = {
         'ip': get_ip_addr(environ),
         'username': username,
         'user_id': user_id,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'repo_store': repo_store,
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
         'detect_force_push': detect_force_push,
         'check_branch_perms': check_branch_perms,
     }
     return scm_data
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
                  initial_call_detection=False, acl_repo_name=None, rc_realm=''):
+        super(BasicAuth, self).__init__(realm=realm, authfunc=authfunc)
         self.realm = realm
         self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
-    def _get_response_from_code(self, http_code):
+    def _get_response_from_code(self, http_code, fallback):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
-            log.exception('Failed to fetch response for code %s', http_code)
-            return HTTPForbidden
+            log.exception('Failed to fetch response class for code %s, using fallback: %s', http_code, fallback)
+            return fallback
 
     def get_rc_realm(self):
         return safe_str(self.rc_realm)
 
     def build_authentication(self):
-        head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
+        header = [('WWW-Authenticate', f'Basic realm="{self.realm}"')]
+
+        # NOTE: the initial_Call detection seems to be not working/not needed witg latest Mercurial
+        # investigate if we still need it.
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
-            custom_response_klass = self._get_response_from_code(
-                self._rc_auth_http_code)
-            return custom_response_klass(headers=head)
-        return HTTPUnauthorized(headers=head)
+            custom_response_klass = self._get_response_from_code(self._rc_auth_http_code, fallback=HTTPUnauthorized)
+            log.debug('Using custom response class: %s', custom_response_klass)
+            return custom_response_klass(headers=header)
+        return HTTPUnauthorized(headers=header)
 
     def authenticate(self, environ):
-        authorization = AUTHORIZATION(environ)
+        authorization = paste.httpheaders.AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
-        (authmeth, auth) = authorization.split(' ', 1)
-        if 'basic' != authmeth.lower():
+        (auth_meth, auth_creds_b64) = authorization.split(' ', 1)
+        if 'basic' != auth_meth.lower():
             return self.build_authentication()
-        auth = auth.strip().decode('base64')
-        _parts = auth.split(':', 1)
+
+        credentials = safe_str(base64.b64decode(auth_creds_b64.strip()))
+        _parts = credentials.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name)
             if auth_data:
                 return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return sha1(
-        config.get('beaker.session.secret', '') +
-        rhodecode.__version__)[:8]
+        config.get(b'beaker.session.secret', b'') + ascii_bytes(rhodecode.__version__)
+    )[:8]
 
 
 def get_current_lang(request):
-    # NOTE(marcink): remove after pyramid move
-    try:
-        return translation.get_lang()[0]
-    except:
-        pass
-
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id=None, is_api=None):
     """
     Attach variables into template context called `c`.
     """
     config = request.registry.settings
 
     rc_config = SettingsModel().get_all_settings(cache=True, from_request=False)
     context.rc_config = rc_config
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     context.rhodecode_edition_id = config.get('rhodecode.edition_id')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.show_revision_number = str2bool(
         rc_config.get('rhodecode_show_revision_number', True))
     context.visual.show_sha_length = safe_int(
         rc_config.get('rhodecode_show_sha_length', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
     context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.ssh_enabled = str2bool(
         config.get('ssh.generate_authorized_keyfile', 'false'))
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
-        config.get('cut_off_limit_diff'))
+        config.get('cut_off_limit_diff'), default=0)
     context.visual.cut_off_limit_file = safe_int(
-        config.get('cut_off_limit_file'))
+        config.get('cut_off_limit_file'), default=0)
 
     context.license = AttributeDict({})
     context.license.hide_license_info = str2bool(
         config.get('license.hide_license_info', False))
 
     # AppEnlight
     context.appenlight_enabled = config.get('appenlight', False)
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     diffmode = {
         "unified": "unified",
         "sideside": "sideside"
     }.get(request.GET.get('diffmode'))
 
     if is_api is not None:
         is_api = hasattr(request, 'rpc_user')
     session_attrs = {
         # defaults
         "clone_url_format": "http",
         "diffmode": "sideside",
         "license_fingerprint": request.session.get('license_fingerprint')
     }
 
     if not is_api:
         # don't access pyramid session for API calls
         if diffmode and diffmode != request.session.get('rc_user_session_attr.diffmode'):
             request.session['rc_user_session_attr.diffmode'] = diffmode
 
         # session settings per user
 
-        for k, v in request.session.items():
+        for k, v in list(request.session.items()):
             pref = 'rc_user_session_attr.'
             if k and k.startswith(pref):
                 k = k[len(pref):]
                 session_attrs[k] = v
 
     context.user_session_attrs = session_attrs
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'session_attrs': session_attrs,
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
     if is_api:
         csrf_token = None
     else:
         csrf_token = auth.get_csrf_token(session=request.session)
 
     context.csrf_token = csrf_token
-    context.backends = rhodecode.BACKENDS.keys()
+    context.backends = list(rhodecode.BACKENDS.keys())
 
     unread_count = 0
     user_bookmark_list = []
     if user_id:
         unread_count = NotificationModel().get_unread_cnt_for_user(user_id)
         user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id)
     context.unread_notifications = unread_count
     context.bookmark_items = user_bookmark_list
 
     # web case
     if hasattr(request, 'user'):
         context.auth_user = request.user
         context.rhodecode_user = request.user
 
     # api case
     if hasattr(request, 'rpc_user'):
         context.auth_user = request.rpc_user
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
     request.set_call_context(context)
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
 
     # make sure that we update permissions each time we call controller
     _auth_token = (
             # ?auth_token=XXX
             request.GET.get('auth_token', '')
             # ?api_key=XXX !LEGACY
             or request.GET.get('api_key', '')
             # or headers....
             or request.headers.get('X-Rc-Auth-Token', '')
     )
     if not _auth_token and request.matchdict:
         url_auth_token = request.matchdict.get('_auth_token')
         _auth_token = url_auth_token
         if _auth_token:
             log.debug('Using URL extracted auth token `...%s`', _auth_token[-4:])
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         # in case someone changes a password for user it triggers session
         # flush and forces a re-login
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user, _auth_token
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
 
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
     config.add_route(name='main_page_repos_data', pattern='/_home_repos')
     config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
     config.add_route(name='hovercard_user',
                      pattern='/_hovercard/user/{user_id}')
 
     config.add_route(name='hovercard_user_group',
                      pattern='/_hovercard/user_group/{user_group_id}')
 
     config.add_route(name='hovercard_pull_request',
                      pattern='/_hovercard/pull_request/{pull_request_id}')
 
     config.add_route(name='hovercard_repo_commit',
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
 def bootstrap_config(request, registry_name='RcTestRegistry'):
     import pyramid.testing
     registry = pyramid.testing.Registry(registry_name)
 
     config = pyramid.testing.setUp(registry=registry, request=request)
 
     # allow pyramid lookup in testing
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
-
+    config.include('rhodecode.lib.rc_cache.archive_cache')
     add_events_routes(config)
 
     return config
 
 
 def bootstrap_request(**kwargs):
     """
     Returns a thin version of Request Object that is used in non-web context like testing/celery
     """
 
     import pyramid.testing
     from rhodecode.lib.request import ThinRequest as _ThinRequest
 
     class ThinRequest(_ThinRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
     class ThinSession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = ThinRequest(**kwargs)
     request.session = ThinSession()
 
     return request
-

rhodecode/lib/caching_query.py

@@ -1,248 +1,251 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """caching_query.py
 
 Represent functions and classes
 which allow the usage of Dogpile caching with SQLAlchemy.
 Introduces a query option called FromCache.
 
 .. versionchanged:: 1.4  the caching approach has been altered to work
    based on a session event.
 
 
 The three new concepts introduced here are:
 
  * ORMCache - an extension for an ORM :class:`.Session`
    retrieves results in/from dogpile.cache.
  * FromCache - a query option that establishes caching
    parameters on a Query
  * RelationshipCache - a variant of FromCache which is specific
    to a query invoked during a lazy load.
 
 The rest of what's here are standard SQLAlchemy and
 dogpile.cache constructs.
 
 """
 from dogpile.cache.api import NO_VALUE
 
 from sqlalchemy import event
 from sqlalchemy.orm import loading
 from sqlalchemy.orm.interfaces import UserDefinedOption
 
 
 DEFAULT_REGION = "sql_cache_short"
 
 
 class ORMCache:
 
     """An add-on for an ORM :class:`.Session` optionally loads full results
     from a dogpile cache region.
 
     cache = ORMCache(regions={})
     cache.listen_on_session(Session)
 
     """
 
     def __init__(self, regions):
         self.cache_regions = regions or self._get_region()
         self._statement_cache = {}
 
     @classmethod
     def _get_region(cls):
         from rhodecode.lib.rc_cache import region_meta
         return region_meta.dogpile_cache_regions
 
     def listen_on_session(self, session_factory):
         event.listen(session_factory, "do_orm_execute", self._do_orm_execute)
 
     def _do_orm_execute(self, orm_context):
-
         for opt in orm_context.user_defined_options:
             if isinstance(opt, RelationshipCache):
                 opt = opt._process_orm_context(orm_context)
                 if opt is None:
                     continue
 
             if isinstance(opt, FromCache):
                 dogpile_region = self.cache_regions[opt.region]
 
+                if dogpile_region.expiration_time <= 0:
+                    # don't cache 0 time expiration cache
+                    continue
+
                 if opt.cache_key:
                     our_cache_key = f'SQL_CACHE_{opt.cache_key}'
                 else:
                     our_cache_key = opt._generate_cache_key(
                         orm_context.statement, orm_context.parameters, self
                     )
 
                 if opt.ignore_expiration:
                     cached_value = dogpile_region.get(
                         our_cache_key,
                         expiration_time=opt.expiration_time,
                         ignore_expiration=opt.ignore_expiration,
                     )
                 else:
 
                     def createfunc():
                         return orm_context.invoke_statement().freeze()
 
                     cached_value = dogpile_region.get_or_create(
                         our_cache_key,
                         createfunc,
                         expiration_time=opt.expiration_time,
                     )
 
                 if cached_value is NO_VALUE:
                     # keyerror?   this is bigger than a keyerror...
                     raise KeyError()
 
                 orm_result = loading.merge_frozen_result(
                     orm_context.session,
                     orm_context.statement,
                     cached_value,
                     load=False,
                 )
                 return orm_result()
 
         else:
             return None
 
     def invalidate(self, statement, parameters, opt):
         """Invalidate the cache value represented by a statement."""
 
         statement = statement.__clause_element__()
 
         dogpile_region = self.cache_regions[opt.region]
 
         cache_key = opt._generate_cache_key(statement, parameters, self)
 
         dogpile_region.delete(cache_key)
 
 
 class FromCache(UserDefinedOption):
     """Specifies that a Query should load results from a cache."""
 
     propagate_to_loaders = False
 
     def __init__(
         self,
         region=DEFAULT_REGION,
         cache_key=None,
         expiration_time=None,
         ignore_expiration=False,
     ):
         """Construct a new FromCache.
 
         :param region: the cache region.  Should be a
          region configured in the dictionary of dogpile
          regions.
 
         :param cache_key: optional.  A string cache key
          that will serve as the key to the query.   Use this
          if your query has a huge amount of parameters (such
          as when using in_()) which correspond more simply to
          some other identifier.
 
         """
         self.region = region
         self.cache_key = cache_key
         self.expiration_time = expiration_time
         self.ignore_expiration = ignore_expiration
 
     # this is not needed as of SQLAlchemy 1.4.28;
     # UserDefinedOption classes no longer participate in the SQL
     # compilation cache key
     def _gen_cache_key(self, anon_map, bindparams):
         return None
 
     def _generate_cache_key(self, statement, parameters, orm_cache):
         """generate a cache key with which to key the results of a statement.
 
         This leverages the use of the SQL compilation cache key which is
         repurposed as a SQL results key.
 
         """
         statement_cache_key = statement._generate_cache_key()
 
         key = statement_cache_key.to_offline_string(
             orm_cache._statement_cache, statement, parameters
         ) + repr(self.cache_key)
         # print("here's our key...%s" % key)
         return key
 
 
 class RelationshipCache(FromCache):
     """Specifies that a Query as called within a "lazy load"
     should load results from a cache."""
 
     propagate_to_loaders = True
 
     def __init__(
         self,
         attribute,
         region=DEFAULT_REGION,
         cache_key=None,
         expiration_time=None,
         ignore_expiration=False,
     ):
         """Construct a new RelationshipCache.
 
         :param attribute: A Class.attribute which
          indicates a particular class relationship() whose
          lazy loader should be pulled from the cache.
 
         :param region: name of the cache region.
 
         :param cache_key: optional.  A string cache key
          that will serve as the key to the query, bypassing
          the usual means of forming a key from the Query itself.
 
         """
         self.region = region
         self.cache_key = cache_key
         self.expiration_time = expiration_time
         self.ignore_expiration = ignore_expiration
         self._relationship_options = {
             (attribute.property.parent.class_, attribute.property.key): self
         }
 
     def _process_orm_context(self, orm_context):
         current_path = orm_context.loader_strategy_path
 
         if current_path:
             mapper, prop = current_path[-2:]
             key = prop.key
 
             for cls in mapper.class_.__mro__:
                 if (cls, key) in self._relationship_options:
                     relationship_option = self._relationship_options[
                         (cls, key)
                     ]
                     return relationship_option
 
     def and_(self, option):
         """Chain another RelationshipCache option to this one.
 
         While many RelationshipCache objects can be specified on a single
         Query separately, chaining them together allows for a more efficient
         lookup during load.
 
         """
         self._relationship_options.update(option._relationship_options)
         return self
 

rhodecode/lib/channelstream.py

@@ -1,372 +1,372 b''
 
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import itsdangerous
 import logging
 import requests
 import datetime
 
 from dogpile.util.readwrite_lock import ReadWriteMutex
-from pyramid.threadlocal import get_current_registry
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.ext_json import json
 from rhodecode.model.db import User
 from rhodecode.lib.str_utils import ascii_str
 from rhodecode.lib.hash_utils import sha1_safe
 
 log = logging.getLogger(__name__)
 
 LOCK = ReadWriteMutex()
 
 USER_STATE_PUBLIC_KEYS = [
     'id', 'username', 'first_name', 'last_name',
     'icon_link', 'display_name', 'display_link']
 
 
 class ChannelstreamException(Exception):
     pass
 
 
 class ChannelstreamConnectionException(ChannelstreamException):
     pass
 
 
 class ChannelstreamPermissionException(ChannelstreamException):
     pass
 
 
 def get_channelstream_server_url(config, endpoint):
     return 'http://{}{}'.format(config['server'], endpoint)
 
 
 def channelstream_request(config, payload, endpoint, raise_exc=True):
     signer = itsdangerous.TimestampSigner(config['secret'])
     sig_for_server = signer.sign(endpoint)
     secret_headers = {'x-channelstream-secret': sig_for_server,
                       'x-channelstream-endpoint': endpoint,
                       'Content-Type': 'application/json'}
     req_url = get_channelstream_server_url(config, endpoint)
 
     log.debug('Sending a channelstream request to endpoint: `%s`', req_url)
     response = None
     try:
         response = requests.post(req_url, data=json.dumps(payload),
                                  headers=secret_headers).json()
     except requests.ConnectionError:
         log.exception('ConnectionError occurred for endpoint %s', req_url)
         if raise_exc:
             raise ChannelstreamConnectionException(req_url)
     except Exception:
         log.exception('Exception related to Channelstream happened')
         if raise_exc:
             raise ChannelstreamConnectionException()
     log.debug('Got channelstream response: %s', response)
     return response
 
 
 def get_user_data(user_id):
     user = User.get(user_id)
     return {
         'id': user.user_id,
         'username': user.username,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'icon_link': h.gravatar_url(user.email, 60),
         'display_name': h.person(user, 'username_or_name_or_email'),
         'display_link': h.link_to_user(user),
         'notifications': user.user_data.get('notification_status', True)
     }
 
 
 def broadcast_validator(channel_name):
     """ checks if user can access the broadcast channel """
     if channel_name == 'broadcast':
         return True
 
 
 def repo_validator(channel_name):
     """ checks if user can access the broadcast channel """
     channel_prefix = '/repo$'
     if channel_name.startswith(channel_prefix):
         elements = channel_name[len(channel_prefix):].split('$')
         repo_name = elements[0]
         can_access = HasRepoPermissionAny(
             'repository.read',
             'repository.write',
             'repository.admin')(repo_name)
         log.debug(
             'permission check for %s channel resulted in %s',
             repo_name, can_access)
         if can_access:
             return True
     return False
 
 
 def check_channel_permissions(channels, plugin_validators, should_raise=True):
     valid_channels = []
 
     validators = [broadcast_validator, repo_validator]
     if plugin_validators:
         validators.extend(plugin_validators)
     for channel_name in channels:
         is_valid = False
         for validator in validators:
             if validator(channel_name):
                 is_valid = True
                 break
         if is_valid:
             valid_channels.append(channel_name)
         else:
             if should_raise:
                 raise ChannelstreamPermissionException()
     return valid_channels
 
 
 def get_channels_info(self, channels):
     payload = {'channels': channels}
     # gather persistence info
     return channelstream_request(self._config(), payload, '/info')
 
 
 def parse_channels_info(info_result, include_channel_info=None):
     """
     Returns data that contains only secure information that can be
     presented to clients
     """
     include_channel_info = include_channel_info or []
 
     user_state_dict = {}
     for userinfo in info_result['users']:
         user_state_dict[userinfo['user']] = {
             k: v for k, v in list(userinfo['state'].items())
             if k in USER_STATE_PUBLIC_KEYS
             }
 
     channels_info = {}
 
     for c_name, c_info in list(info_result['channels'].items()):
         if c_name not in include_channel_info:
             continue
         connected_list = []
         for username in c_info['users']:
             connected_list.append({
                 'user': username,
                 'state': user_state_dict[username]
             })
         channels_info[c_name] = {'users': connected_list,
                                  'history': c_info['history']}
 
     return channels_info
 
 
 def log_filepath(history_location, channel_name):
 
     channel_hash = ascii_str(sha1_safe(channel_name))
     filename = f'{channel_hash}.log'
     filepath = os.path.join(history_location, filename)
     return filepath
 
 
 def read_history(history_location, channel_name):
     filepath = log_filepath(history_location, channel_name)
     if not os.path.exists(filepath):
         return []
     history_lines_limit = -100
     history = []
     with open(filepath, 'rb') as f:
         for line in f.readlines()[history_lines_limit:]:
             try:
                 history.append(json.loads(line))
             except Exception:
                 log.exception('Failed to load history')
     return history
 
 
 def update_history_from_logs(config, channels, payload):
     history_location = config.get('history.location')
     for channel in channels:
         history = read_history(history_location, channel)
         payload['channels_info'][channel]['history'] = history
 
 
 def write_history(config, message):
     """ writes a message to a base64encoded filename """
     history_location = config.get('history.location')
     if not os.path.exists(history_location):
         return
     try:
         LOCK.acquire_write_lock()
         filepath = log_filepath(history_location, message['channel'])
         json_message = json.dumps(message)
         with open(filepath, 'ab') as f:
             f.write(json_message)
             f.write('\n')
     finally:
         LOCK.release_write_lock()
 
 
 def get_connection_validators(registry):
     validators = []
     for k, config in list(registry.rhodecode_plugins.items()):
         validator = config.get('channelstream', {}).get('connect_validator')
         if validator:
             validators.append(validator)
     return validators
 
 
 def get_channelstream_config(registry=None):
     if not registry:
+        from pyramid.threadlocal import get_current_registry
         registry = get_current_registry()
 
     rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
     channelstream_config = rhodecode_plugins.get('channelstream', {})
     return channelstream_config
 
 
 def post_message(channel, message, username, registry=None):
     channelstream_config = get_channelstream_config(registry)
     if not channelstream_config.get('enabled'):
         return
 
     message_obj = message
     if isinstance(message, str):
         message_obj = {
             'message': message,
             'level': 'success',
             'topic': '/notifications'
         }
 
     log.debug('Channelstream: sending notification to channel %s', channel)
     payload = {
         'type': 'message',
         'timestamp': datetime.datetime.utcnow(),
         'user': 'system',
         'exclude_users': [username],
         'channel': channel,
         'message': message_obj
     }
 
     try:
         return channelstream_request(
             channelstream_config, [payload], '/message',
             raise_exc=False)
     except ChannelstreamException:
         log.exception('Failed to send channelstream data')
         raise
 
 
 def _reload_link(label):
     return (
         '<a onclick="window.location.reload()">'
         '<strong>{}</strong>'
         '</a>'.format(label)
     )
 
 
 def pr_channel(pull_request):
     repo_name = pull_request.target_repo.repo_name
     pull_request_id = pull_request.pull_request_id
     channel = '/repo${}$/pr/{}'.format(repo_name, pull_request_id)
     log.debug('Getting pull-request channelstream broadcast channel: %s', channel)
     return channel
 
 
 def comment_channel(repo_name, commit_obj=None, pull_request_obj=None):
     channel = None
     if commit_obj:
         channel = '/repo${}$/commit/{}'.format(
             repo_name, commit_obj.raw_id
         )
     elif pull_request_obj:
         channel = '/repo${}$/pr/{}'.format(
             repo_name, pull_request_obj.pull_request_id
         )
     log.debug('Getting comment channelstream broadcast channel: %s', channel)
 
     return channel
 
 
 def pr_update_channelstream_push(request, pr_broadcast_channel, user, msg, **kwargs):
     """
     Channel push on pull request update
     """
     if not pr_broadcast_channel:
         return
 
     _ = request.translate
 
     message = '{} {}'.format(
         msg,
         _reload_link(_(' Reload page to load changes')))
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         pr_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
 
 def comment_channelstream_push(request, comment_broadcast_channel, user, msg, **kwargs):
     """
     Channelstream push on comment action, on commit, or pull-request
     """
     if not comment_broadcast_channel:
         return
 
     _ = request.translate
 
     comment_data = kwargs.pop('comment_data', {})
     user_data = kwargs.pop('user_data', {})
     comment_id = list(comment_data.keys())[0] if comment_data else ''
 
     message = '<strong>{}</strong> {} #{}'.format(
         user.username,
         msg,
         comment_id,
     )
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
     message_obj = {
         'message': None,
         'user': user.username,
         'comment_id': comment_id,
         'comment_data': comment_data,
         'user_data': user_data,
         'topic': '/comment'
     }
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)

rhodecode/lib/codeblocks.py

@@ -1,797 +1,819 b''
 
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
-from itertools import groupby
+import itertools
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from pygments.lexers.special import TextLexer, Token
 from pygments.lexers import get_lexer_by_name
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
-from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict, safe_unicode
+from rhodecode.lib.str_utils import safe_str
+from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
+from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer, DEL_FILENODE, BIN_FILENODE
 
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
-    log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
+    log.debug('Generating file node pygment tokens for %s, file=`%s`, org_lexer:%s',
               lexer, filenode, org_lexer)
-    content = filenode.content
+    content = filenode.str_content
     tokens = tokenize_string(content, lexer)
     lines = split_token_stream(tokens, content)
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
 
+    # pygments needs to operate on str
+    str_content = safe_str(content)
+
     if isinstance(lexer, TextLexer):
-        lexed = [(Token.Text, content)]
+        # we convert content here to STR because pygments does that while tokenizing
+        # if we DON'T get a lexer for unknown file type
+        lexed = [(Token.Text, str_content)]
     else:
-        lexed = lex(content, lexer)
+        lexed = lex(str_content, lexer)
 
     for token_type, token_text in lexed:
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens, content):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     token_buffer = []
+
     for token_class, token_text in tokens:
+
+        # token_text, should be str
         parts = token_text.split('\n')
         for part in parts[:-1]:
             token_buffer.append((token_class, part))
             yield token_buffer
             token_buffer = []
 
         token_buffer.append((token_class, parts[-1]))
 
     if token_buffer:
         yield token_buffer
     elif content:
         # this is a special case, we have the content, but tokenization didn't produce
-        # any results. THis can happen if know file extensions like .css have some bogus
+        # any results. This can happen if know file extensions like .css have some bogus
         # unicode content without any newline characters
         yield [(pygment_token_class(Token.Text), content)]
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
-    grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
+    grouped_annotations_lines = itertools.groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
-            result.append('<span class="%s">' % token_class)
+            result.append(f'<span class="{token_class}">')
         else:
             result.append('<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
-                result.append('<%s>' % op_tag)
+                result.append(f'<{op_tag}>')
 
             # NOTE(marcink): in some cases of mixed encodings, we might run into
             # troubles in the html_escape, in this case we say unicode force on token_text
             # that would ensure "correct" data even with the cost of rendered
             try:
                 escaped_text = html_escape(token_text)
             except TypeError:
-                escaped_text = html_escape(safe_unicode(token_text))
+                escaped_text = html_escape(safe_str(token_text))
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
-                result.append('</%s>' % op_tag)
+                result.append(f'</{op_tag}>')
 
         result.append('</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
-    for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
+    for token_class, op_list in itertools.groupby(tokenstream, lambda t: t[0]):
         ops = []
-        for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
+        for token_op, token_text_list in itertools.groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
+
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
-    similarity = difflib.SequenceMatcher(None,
+    def int_convert(val):
+        if isinstance(val, int):
+            return str(val)
+        return val
+
+    similarity = difflib.SequenceMatcher(
+        None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
-    if similarity < 0.6: # return, the blocks are too different
+    if similarity < 0.6:  # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
-    token_sequence_matcher = difflib.SequenceMatcher(None,
+    token_sequence_matcher = difflib.SequenceMatcher(
+        None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
-        # check the differences by token block types first to give a more
+        # check the differences by token block types first to give a
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
-                old_tokens_result.append((token_class, 'del', token_text))
+                old_tokens_result.append((token_class, 'del', int_convert(token_text)))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
-                new_tokens_result.append((token_class, 'ins', token_text))
+                new_tokens_result.append((token_class, 'ins', int_convert(token_text)))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
-                for char in token_text:
+                for char in map(lambda i: i, token_text):
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
-                for char in token_text:
+                for char in map(lambda i: i, token_text):
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
-                token_class, token_text in old_char_tokens])
+                                  token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
-                token_class, token_text in new_char_tokens])
+                                  token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11  # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
-                            obuffer.append((old_char_tokens[a+i][0], 'del', c))
+                            obuffer.append((old_char_tokens[a+i][0], 'del', int_convert(c)))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
-                            nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
+                            nbuffer.append((new_char_tokens[b+i][0], 'ins', int_convert(c)))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
-                            obuffer.append((token_class, 'del', token_text))
+                            obuffer.append((token_class, 'del', int_convert(token_text)))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
-                            nbuffer.append((token_class, 'ins', token_text))
+                            nbuffer.append((token_class, 'ins', int_convert(token_text)))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
-                            obuffer.append((token_class, 'del', token_text))
+                            obuffer.append((token_class, 'del', int_convert(token_text)))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
-                            nbuffer.append((token_class, 'ins', token_text))
+                            nbuffer.append((token_class, 'ins', int_convert(token_text)))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 def diffset_node_getter(commit):
-    def get_node(fname):
+    def get_diff_node(file_name):
+
         try:
-            return commit.get_node(fname)
+            return commit.get_node(file_name, pre_load=['size', 'flags', 'data'])
         except NodeDoesNotExistError:
             return None
 
-    return get_node
+    return get_diff_node
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL'  # highlights using original file, slow
     HL_FAST = 'FAST'  # highlights using just the line, fast but not correct
                       # in the case of multiline code
     HL_NONE = 'NONE'  # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_repo_name=None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  # files over this size will use fast highlighting
                  max_file_size_limit=150 * 1024,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {
             'before': {},
             'after': {}
         }
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.target_repo_name = target_repo_name or repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             target_repo_name=self.target_repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = StrictAttributeDict(dict(
                 source_ref=diffset.source_ref,
                 target_ref=diffset.target_ref,
                 repo_name=diffset.repo_name,
                 source_repo_name=diffset.source_repo_name,
                 target_repo_name=diffset.target_repo_name,
             ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r', patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             node_hl_mode = self.HL_NONE if patch['chunks'] == [] else None
             hl_mode = node_hl_mode or self.highlight_mode
 
             if hl_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif hl_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
         raw_id_uid = ''
         if self.source_nodes.get(source_filename):
             raw_id_uid = self.source_nodes[source_filename].commit.raw_id
 
         if not raw_id_uid and self.target_nodes.get(target_filename):
             # in case this is a new file we only have it in target
             raw_id_uid = self.target_nodes[target_filename].commit.raw_id
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': patch['is_limited_diff'],
             'hunks': [],
             'hunk_ops': None,
             'diffset': self,
             'raw_id': raw_id_uid,
         })
 
         file_chunks = patch['chunks'][1:]
         for i, hunk in enumerate(file_chunks, 1):
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             hunkbit.index = i
             filediff.hunks.append(hunkbit)
 
         # Simulate hunk on OPS type line which doesn't really contain any diff
         # this allows commenting on those
         if not file_chunks:
             actions = []
-            for op_id, op_text in filediff.patch['stats']['ops'].items():
+            for op_id, op_text in list(filediff.patch['stats']['ops'].items()):
                 if op_id == DEL_FILENODE:
                     actions.append('file was removed')
                 elif op_id == BIN_FILENODE:
                     actions.append('binary diff hidden')
                 else:
-                    actions.append(safe_unicode(op_text))
+                    actions.append(safe_str(op_text))
             action_line = 'NO CONTENT: ' + \
                           ', '.join(actions) or 'UNDEFINED_ACTION'
 
             hunk_ops = {'source_length': 0, 'source_start': 0,
                         'lines': [
                             {'new_lineno': 0, 'old_lineno': 1,
                              'action': 'unmod-no-hl', 'line': action_line}
                         ],
                         'section_header': '', 'target_start': 1, 'target_length': 1}
 
             hunkbit = self.parse_hunk(hunk_ops, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunk_ops = hunkbit
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
+
             if line['action'] in ['unmod', 'unmod-no-hl']:
                 no_hl = line['action'] == 'unmod-no-hl'
-                result.lines.extend(
-                    self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
+                parsed_lines = self.parse_lines(before, after, source_file, target_file, no_hl=no_hl)
+                result.lines.extend(parsed_lines)
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
+                #line['line'] = safe_str(line['line'])
             elif line['action'] == 'new-no-nl':
+                #line['line'] = safe_str(line['line'])
                 after.append(line)
 
         all_actions = [x['action'] for x in after] + [x['action'] for x in before]
         no_hl = {x for x in all_actions} == {'unmod-no-hl'}
-        result.lines.extend(
-            self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
-        # NOTE(marcink): we must keep list() call here so we can cache the result...
+        parsed_no_hl_lines = self.parse_lines(before, after, source_file, target_file, no_hl=no_hl)
+        result.lines.extend(parsed_no_hl_lines)
+
+        # NOTE(marcink): we must keep list() call here, so we can cache the result...
         result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file,
                     no_hl=False):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
 
         before_newline = AttributeDict()
         after_newline = AttributeDict()
         if before_lines and before_lines[-1]['action'] == 'old-no-nl':
             before_newline_line = before_lines.pop(-1)
             before_newline.content = '\n {}'.format(
                 render_tokenstream(
-                    [(x[0], '', x[1])
+                    [(x[0], '', safe_str(x[1]))
                      for x in [('nonl', before_newline_line['line'])]]))
 
         if after_lines and after_lines[-1]['action'] == 'new-no-nl':
             after_newline_line = after_lines.pop(-1)
             after_newline.content = '\n {}'.format(
                 render_tokenstream(
-                    [(x[0], '', x[1])
+                    [(x[0], '', safe_str(x[1]))
                      for x in [('nonl', after_newline_line['line'])]]))
 
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
-                    before_tokens = [('nonl', before['line'])]
+                    before_tokens = [('nonl', safe_str(before['line']))]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'], line_number=before['old_lineno'],
                         input_file=source_file, no_hl=no_hl, source='before')
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
 
                 original.get_comment_args = (
                     source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
-                    after_tokens = [('nonl', after['line'])]
+                    after_tokens = [('nonl', safe_str(after['line']))]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         input_file=target_file, no_hl=no_hl, source='after')
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
 
                 modified.get_comment_args = (target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
             if not before_lines and before_newline:
                 original.content += before_newline.content
                 before_newline = None
             if not after_lines and after_newline:
                 modified.content += after_newline.content
                 after_newline = None
 
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_line_tokens(self, line_text, line_number, input_file=None, no_hl=False, source=''):
         filenode = None
         filename = None
 
         if isinstance(input_file, str):
             filename = input_file
         elif isinstance(input_file, FileNode):
             filenode = input_file
-            filename = input_file.unicode_path
+            filename = input_file.str_path
 
         hl_mode = self.HL_NONE if no_hl else self.highlight_mode
         if hl_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
-            file_size_allowed = input_file.size < self.max_file_size_limit
+            file_size_allowed = filenode.size < self.max_file_size_limit
             if line_number and file_size_allowed:
-                return self.get_tokenized_filenode_line(input_file, line_number, lexer, source)
+                return self.get_tokenized_filenode_line(filenode, line_number, lexer, source)
 
         if hl_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None, source=''):
+        name_hash = hash(filenode)
 
-        def tokenize(_filenode):
-            self.highlighted_filenodes[source][filenode] = filenode_as_lines_tokens(filenode, lexer)
+        hl_node_code = self.highlighted_filenodes[source]
 
-        if filenode not in self.highlighted_filenodes[source]:
-            tokenize(filenode)
+        if name_hash not in hl_node_code:
+            hl_node_code[name_hash] = filenode_as_lines_tokens(filenode, lexer)
 
         try:
-            return self.highlighted_filenodes[source][filenode][line_number - 1]
+            return hl_node_code[name_hash][line_number - 1]
         except Exception:
-            log.exception('diff rendering error')
+            log.exception('diff rendering error on L:%s and file=%s', line_number - 1, filenode.name)
             return [('', 'L{}: rhodecode diff rendering error'.format(line_number))]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'unmod-no-hl': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.get_comment_args)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/lib/db_manage.py

@@ -1,679 +1,688 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
-    print(('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper())
+    print((('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()))
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
-                 SESSION=None, cli_args=None):
+                 SESSION=None, cli_args=None, enc_key=b''):
+
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.cli_args = cli_args or {}
+        self.sa = None
+        self.engine = None
+        self.enc_key = enc_key
+        # sets .sa .engine
         self.init_db(SESSION=SESSION)
+
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def db_exists(self):
         if not self.sa:
             self.init_db()
         try:
             self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
             return True
         except Exception:
             return False
         finally:
             self.sa.rollback()
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
+
         if SESSION:
             self.sa = SESSION
+            self.engine = SESSION.bind
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
-            init_model(engine)
+            init_model(engine, encryption_key=self.enc_key)
             self.sa = Session()
+            self.engine = engine
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
+        engine = self.engine
+
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('db tables bootstrap: Nothing done.')
             sys.exit(0)
         if destroy:
-            Base.metadata.drop_all()
+            Base.metadata.drop_all(bind=engine)
 
         checkfirst = not override
-        Base.metadata.create_all(checkfirst=checkfirst)
+        Base.metadata.create_all(bind=engine, checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def run_post_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
-        from rhodecode.lib.dbmigrate.migrate.exceptions import \
-            DatabaseNotControlledError
+        from rhodecode.lib.dbmigrate.migrate.exceptions import DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         if version:
             DbMigrateVersion.set_version(version)
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
-            msg = ('Found current database db_uri under version '
-                   'control with version {}'.format(curr_version))
+            msg = (f'Found current database db_uri under version '
+                   f'control with version {curr_version}')
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
-            msg = ('Current database is not under version control. Setting '
-                   'as version %s' % curr_version)
+            msg = f'Current database is not under version control. ' \
+                  f'Setting as version {curr_version}'
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
-
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
-        upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
-        notify('attempting to upgrade database from '
-               'version %s to version %s' % (curr_version, __dbversion__))
+        upgrade_steps = list(range(curr_version + 1, __dbversion__ + 1))
+        notify(f'attempting to upgrade database from '
+               f'version {curr_version} to version {__dbversion__}')
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
-            notify('performing upgrade step %s' % step)
+            notify(f'performing upgrade step {step}')
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
-            notify('schema upgrade for step %s completed' % (step,))
+            notify(f'schema upgrade for step {step} completed')
 
             _step = step
 
         self.run_post_migration_tasks()
-        notify('upgrade to version %s successful' % _step)
+        notify(f'upgrade to version {step} successful')
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
-                .filter(User.username == User.DEFAULT_USER)\
-                .one()
+            .filter(User.username == User.DEFAULT_USER)\
+            .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
             username = eval(input('Specify admin username:'))
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
             email = eval(input('Specify admin email:'))
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution.exchange'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'topic'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
         defaults = [
             ('auth_plugins',
              'egg:rhodecode-enterprise-ce#token,egg:rhodecode-enterprise-ce#rhodecode',
              'list'),
 
             ('auth_authtoken_enabled',
              'True',
              'bool'),
 
             ('auth_rhodecode_enabled',
              'True',
              'bool'),
         ]
         for k, v, t in defaults:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = eval(input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             ))
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
 
-            q = ('Given path %s is not writeable, do you want to '
-                 'continue with read only mode ? [y/n]' % (path,))
+            q = (f'Given path {path} is not writeable, do you want to '
+                 f'continue with read only mode ? [y/n]')
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
-            q = ('Path looks like a symlink, RhodeCode Enterprise will store '
-                 'given path as %s ? [y/n]') % (real_path,)
+            q = (f'Path looks like a symlink, RhodeCode Enterprise will store '
+                 f'given path as {real_path} ? [y/n]')
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
 
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
             ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
 
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
 
             # VCS Settings
             ('pr_merge_enabled', True, 'bool'),
             ('use_outdated_comments', True, 'bool'),
             ('diff_cache', True, 'bool'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`', username)
         user = UserModel().create_or_update(
             username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
                 description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
                                             firstname='Anonymous',
                                             lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
         # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/exceptions.py

@@ -1,182 +1,202 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Set of custom exceptions used in RhodeCode
 """
 
 from webob.exc import HTTPClientError
 from pyramid.httpexceptions import HTTPBadGateway
 
 
 class LdapUsernameError(Exception):
     pass
 
 
 class LdapPasswordError(Exception):
     pass
 
 
 class LdapConnectionError(Exception):
     pass
 
 
 class LdapImportError(Exception):
     pass
 
 
 class DefaultUserException(Exception):
     pass
 
 
 class UserOwnsReposException(Exception):
     pass
 
 
 class UserOwnsRepoGroupsException(Exception):
     pass
 
 
 class UserOwnsUserGroupsException(Exception):
     pass
 
 
 class UserOwnsPullRequestsException(Exception):
     pass
 
 
 class UserOwnsArtifactsException(Exception):
     pass
 
 
 class UserGroupAssignedException(Exception):
     pass
 
 
 class StatusChangeOnClosedPullRequestError(Exception):
     pass
 
 
 class AttachedForksError(Exception):
     pass
 
 
 class AttachedPullRequestsError(Exception):
     pass
 
 
 class RepoGroupAssignmentError(Exception):
     pass
 
 
 class NonRelativePathError(Exception):
     pass
 
 
 class HTTPRequirementError(HTTPClientError):
     title = explanation = 'Repository Requirement Missing'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super(HTTPRequirementError, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class HTTPLockedRC(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can
     be overwritten by _code keyword argument passed into constructors
     """
     code = 423
     title = explanation = 'Repository Locked'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
-        from rhodecode import CONFIG
-        from rhodecode.lib.utils2 import safe_int
-        _code = CONFIG.get('lock_ret_code')
-        self.code = safe_int(_code, self.code)
+        import rhodecode
+
+        self.code = rhodecode.ConfigGet().get_int('lock_ret_code', missing=self.code)
+
         self.title = self.explanation = message
         super(HTTPLockedRC, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class HTTPBranchProtected(HTTPClientError):
     """
     Special Exception For Indicating that branch is protected in RhodeCode, the
     return code can be overwritten by _code keyword argument passed into constructors
     """
     code = 403
     title = explanation = 'Branch Protected'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super(HTTPBranchProtected, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class IMCCommitError(Exception):
     pass
 
 
 class UserCreationError(Exception):
     pass
 
 
 class NotAllowedToCreateUserError(Exception):
     pass
 
 
 class RepositoryCreationError(Exception):
     pass
 
 
 class VCSServerUnavailable(HTTPBadGateway):
     """ HTTP Exception class for VCS Server errors """
     code = 502
     title = 'VCS Server Error'
     causes = [
         'VCS Server is not running',
         'Incorrect vcs.server=host:port',
         'Incorrect vcs.server.protocol',
     ]
 
     def __init__(self, message=''):
         self.explanation = 'Could not connect to VCS Server'
         if message:
             self.explanation += ': ' + message
         super(VCSServerUnavailable, self).__init__()
 
 
 class ArtifactMetadataDuplicate(ValueError):
 
     def __init__(self, *args, **kwargs):
         self.err_section = kwargs.pop('err_section', None)
         self.err_key = kwargs.pop('err_key', None)
         super(ArtifactMetadataDuplicate, self).__init__(*args, **kwargs)
 
 
 class ArtifactMetadataBadValueType(ValueError):
     pass
 
 
 class CommentVersionMismatch(ValueError):
     pass
+
+
+class SignatureVerificationError(ValueError):
+    pass
+
+
+def signature_verification_error(msg):
+    details = """
+Encryption signature verification failed. 
+Please check your value of secret key, and/or encrypted value stored. 
+Secret key stored inside .ini file:
+`rhodecode.encrypted_values.secret` or defaults to
+`beaker.session.secret` 
+
+Probably the stored values were encrypted using a different secret then currently set in .ini file    
+"""
+
+    final_msg = f'{msg}\n{details}'
+    return SignatureVerificationError(final_msg)
+

rhodecode/lib/feedgenerator/feedgenerator.py

@@ -1,444 +1,443 b''
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
 #     1. Redistributions of source code must retain the above copyright notice,
 #        this list of conditions and the following disclaimer.
 #
 #     2. Redistributions in binary form must reproduce the above copyright
 #        notice, this list of conditions and the following disclaimer in the
 #        documentation and/or other materials provided with the distribution.
 #
 #     3. Neither the name of Django nor the names of its contributors may be used
 #        to endorse or promote products derived from this software without
 #        specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """
 For definitions of the different versions of RSS, see:
 http://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004/02/04/incompatible-rss
 """
 
 
 import datetime
 import io
 
-import pytz
 from six.moves.urllib import parse as urlparse
 
 from rhodecode.lib.feedgenerator import datetime_safe
 from rhodecode.lib.feedgenerator.utils import SimplerXMLGenerator, iri_to_uri, force_text
 
 
 #### The following code comes from ``django.utils.feedgenerator`` ####
 
 
 def rfc2822_date(date):
     # We can't use strftime() because it produces locale-dependent results, so
     # we have to map english month and day names manually
     months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',)
     days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     # We do this ourselves to be timezone aware, email.Utils is not tz aware.
     dow = days[date.weekday()]
     month = months[date.month - 1]
     time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month))
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + '-0000'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d%02d' % (hour, minute)
 
 
 def rfc3339_date(date):
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     time_str = date.strftime('%Y-%m-%dT%H:%M:%S')
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + 'Z'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d:%02d' % (hour, minute)
 
 
 def get_tag_uri(url, date):
     """
     Creates a TagURI.
 
     See http://web.archive.org/web/20110514113830/http://diveintomark.org/archives/2004/05/28/howto-atom-id
     """
     bits = urlparse(url)
     d = ''
     if date is not None:
         d = ',%s' % datetime_safe.new_datetime(date).strftime('%Y-%m-%d')
     return 'tag:%s%s:%s/%s' % (bits.hostname, d, bits.path, bits.fragment)
 
 
 class SyndicationFeed(object):
     """Base class for all syndication feeds. Subclasses should provide write()"""
 
     def __init__(self, title, link, description, language=None, author_email=None,
                  author_name=None, author_link=None, subtitle=None, categories=None,
                  feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs):
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [force_text(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         self.feed = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'language': to_unicode(language),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'subtitle': to_unicode(subtitle),
             'categories': categories or (),
             'feed_url': iri_to_uri(feed_url),
             'feed_copyright': to_unicode(feed_copyright),
             'id': feed_guid or link,
             'ttl': ttl,
         }
         self.feed.update(kwargs)
         self.items = []
 
     def add_item(self, title, link, description, author_email=None,
                  author_name=None, author_link=None, pubdate=None, comments=None,
                  unique_id=None, unique_id_is_permalink=None, enclosure=None,
                  categories=(), item_copyright=None, ttl=None, updateddate=None,
                  enclosures=None, **kwargs):
         """
         Adds an item to the feed. All args are expected to be Python Unicode
         objects except pubdate and updateddate, which are datetime.datetime
         objects, and enclosures, which is an iterable of instances of the
         Enclosure class.
         """
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [to_unicode(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         if enclosure is None:
             enclosures = [] if enclosures is None else enclosures
 
         item = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'pubdate': pubdate,
             'updateddate': updateddate,
             'comments': to_unicode(comments),
             'unique_id': to_unicode(unique_id),
             'unique_id_is_permalink': unique_id_is_permalink,
             'enclosures': enclosures,
             'categories': categories or (),
             'item_copyright': to_unicode(item_copyright),
             'ttl': ttl,
         }
         item.update(kwargs)
         self.items.append(item)
 
     def num_items(self):
         return len(self.items)
 
     def root_attributes(self):
         """
         Return extra attributes to place on the root (i.e. feed/channel) element.
         Called from write().
         """
         return {}
 
     def add_root_elements(self, handler):
         """
         Add elements in the root (i.e. feed/channel) element. Called
         from write().
         """
         pass
 
     def item_attributes(self, item):
         """
         Return extra attributes to place on each item (i.e. item/entry) element.
         """
         return {}
 
     def add_item_elements(self, handler, item):
         """
         Add elements on each item (i.e. item/entry) element.
         """
         pass
 
     def write(self, outfile, encoding):
         """
         Outputs the feed in the given encoding to outfile, which is a file-like
         object. Subclasses should override this.
         """
         raise NotImplementedError('subclasses of SyndicationFeed must provide a write() method')
 
     def writeString(self, encoding):
         """
         Returns the feed in the given encoding as a string.
         """
         s = io.StringIO()
         self.write(s, encoding)
         return s.getvalue()
 
     def latest_post_date(self):
         """
         Returns the latest item's pubdate or updateddate. If no items
         have either of these attributes this returns the current UTC date/time.
         """
         latest_date = None
         date_keys = ('updateddate', 'pubdate')
 
         for item in self.items:
             for date_key in date_keys:
                 item_date = item.get(date_key)
                 if item_date:
                     if latest_date is None or item_date > latest_date:
                         latest_date = item_date
 
         # datetime.now(tz=utc) is slower, as documented in django.utils.timezone.now
-        return latest_date or datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
+        return latest_date or datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
 
 
 class Enclosure(object):
     """Represents an RSS enclosure"""
     def __init__(self, url, length, mime_type):
         """All args are expected to be Python Unicode objects"""
         self.length, self.mime_type = length, mime_type
         self.url = iri_to_uri(url)
 
 
 class RssFeed(SyndicationFeed):
     content_type = 'application/rss+xml; charset=utf-8'
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement("rss", self.rss_attributes())
         handler.startElement("channel", self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         self.endChannelElement(handler)
         handler.endElement("rss")
 
     def rss_attributes(self):
         return {"version": self._version,
                 "xmlns:atom": "http://www.w3.org/2005/Atom"}
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement('item', self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("item")
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", self.feed['link'])
         handler.addQuickElement("description", self.feed['description'])
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("atom:link", None, {"rel": "self", "href": self.feed['feed_url']})
         if self.feed['language'] is not None:
             handler.addQuickElement("language", self.feed['language'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", cat)
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("copyright", self.feed['feed_copyright'])
         handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()))
         if self.feed['ttl'] is not None:
             handler.addQuickElement("ttl", self.feed['ttl'])
 
     def endChannelElement(self, handler):
         handler.endElement("channel")
 
 
 class RssUserland091Feed(RssFeed):
     _version = "0.91"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
 
 class Rss201rev2Feed(RssFeed):
     # Spec: http://blogs.law.harvard.edu/tech/rss
     _version = "2.0"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
         # Author information.
         if item["author_name"] and item["author_email"]:
             handler.addQuickElement("author", "%s (%s)" % (item['author_email'], item['author_name']))
         elif item["author_email"]:
             handler.addQuickElement("author", item["author_email"])
         elif item["author_name"]:
             handler.addQuickElement(
                 "dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}
             )
 
         if item['pubdate'] is not None:
             handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']))
         if item['comments'] is not None:
             handler.addQuickElement("comments", item['comments'])
         if item['unique_id'] is not None:
             guid_attrs = {}
             if isinstance(item.get('unique_id_is_permalink'), bool):
                 guid_attrs['isPermaLink'] = str(item['unique_id_is_permalink']).lower()
             handler.addQuickElement("guid", item['unique_id'], guid_attrs)
         if item['ttl'] is not None:
             handler.addQuickElement("ttl", item['ttl'])
 
         # Enclosure.
         if item['enclosures']:
             enclosures = list(item['enclosures'])
             if len(enclosures) > 1:
                 raise ValueError(
                     "RSS feed items may only have one enclosure, see "
                     "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
                 )
             enclosure = enclosures[0]
             handler.addQuickElement('enclosure', '', {
                 'url': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", cat)
 
 
 class Atom1Feed(SyndicationFeed):
     # Spec: https://tools.ietf.org/html/rfc4287
     content_type = 'application/atom+xml; charset=utf-8'
     ns = "http://www.w3.org/2005/Atom"
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement('feed', self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         handler.endElement("feed")
 
     def root_attributes(self):
         if self.feed['language'] is not None:
             return {"xmlns": self.ns, "xml:lang": self.feed['language']}
         else:
             return {"xmlns": self.ns}
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", "", {"rel": "alternate", "href": self.feed['link']})
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']})
         handler.addQuickElement("id", self.feed['id'])
         handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()))
         if self.feed['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", self.feed['author_name'])
             if self.feed['author_email'] is not None:
                 handler.addQuickElement("email", self.feed['author_email'])
             if self.feed['author_link'] is not None:
                 handler.addQuickElement("uri", self.feed['author_link'])
             handler.endElement("author")
         if self.feed['subtitle'] is not None:
             handler.addQuickElement("subtitle", self.feed['subtitle'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", "", {"term": cat})
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("rights", self.feed['feed_copyright'])
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement("entry", self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("entry")
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"})
 
         if item['pubdate'] is not None:
             handler.addQuickElement('published', rfc3339_date(item['pubdate']))
 
         if item['updateddate'] is not None:
             handler.addQuickElement('updated', rfc3339_date(item['updateddate']))
 
         # Author information.
         if item['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", item['author_name'])
             if item['author_email'] is not None:
                 handler.addQuickElement("email", item['author_email'])
             if item['author_link'] is not None:
                 handler.addQuickElement("uri", item['author_link'])
             handler.endElement("author")
 
         # Unique ID.
         if item['unique_id'] is not None:
             unique_id = item['unique_id']
         else:
             unique_id = get_tag_uri(item['link'], item['pubdate'])
         handler.addQuickElement("id", unique_id)
 
         # Summary.
         if item['description'] is not None:
             handler.addQuickElement("summary", item['description'], {"type": "html"})
 
         # Enclosures.
         for enclosure in item['enclosures']:
             handler.addQuickElement('link', '', {
                 'rel': 'enclosure',
                 'href': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", "", {"term": cat})
 
         # Rights.
         if item['item_copyright'] is not None:
             handler.addQuickElement("rights", item['item_copyright'])
 
 
 # This isolates the decision of what the system default is, so calling code can
 # do "feedgenerator.DefaultFeed" instead of "feedgenerator.Rss201rev2Feed".
 DefaultFeed = Rss201rev2Feed
  No newline at end of file

rhodecode/lib/index/__init__.py

@@ -1,155 +1,155 b''
 
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 import importlib
 import logging
 
 from rhodecode.lib.index.search_utils import normalize_text_for_matching
 
 log = logging.getLogger(__name__)
 
 # leave defaults for backward compat
 default_searcher = 'rhodecode.lib.index.whoosh'
 default_location = '%(here)s/data/index'
 
 ES_VERSION_2 = '2'
 ES_VERSION_6 = '6'
 # for legacy reasons we keep 2 compat as default
 DEFAULT_ES_VERSION = ES_VERSION_2
 
 try:
     from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG  # pragma: no cover
 except ImportError:
     log.warning('rhodecode_tools not available, use of full text search is limited')
     pass
 
 
 class BaseSearcher(object):
     query_lang_doc = ''
     es_version = None
     name = None
     DIRECTION_ASC = 'asc'
     DIRECTION_DESC = 'desc'
 
     def __init__(self):
         pass
 
     def cleanup(self):
         pass
 
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                raise_on_exc=True):
         raise Exception('NotImplemented')
 
     @staticmethod
     def query_to_mark(query, default_field=None):
         """
         Formats the query to mark token for jquery.mark.js highlighting. ES could
         have a different format optionally.
 
         :param default_field:
         :param query:
         """
         return ' '.join(normalize_text_for_matching(query).split())
 
     @property
     def is_es_6(self):
         return self.es_version == ES_VERSION_6
 
     def get_handlers(self):
         return {}
 
     @staticmethod
     def extract_search_tags(query):
         return []
 
     @staticmethod
     def escape_specials(val):
         """
         Handle and escape reserved chars for search
         """
         return val
 
     def sort_def(self, search_type, direction, sort_field):
         """
         Defines sorting for search. This function should decide if for given
         search_type, sorting can be done with sort_field.
 
         It also should translate common sort fields into backend specific. e.g elasticsearch
         """
         raise NotImplementedError()
 
     @staticmethod
     def get_sort(search_type, search_val):
         """
         Method used to parse the GET search sort value to a field and direction.
         e.g asc:lines == asc, lines
 
         There's also a legacy support for newfirst/oldfirst which defines commit
         sorting only
         """
 
         direction = BaseSearcher.DIRECTION_ASC
         sort_field = None
 
         if not search_val:
             return direction, sort_field
 
         if search_val.startswith('asc:'):
             sort_field = search_val[4:]
             direction = BaseSearcher.DIRECTION_ASC
         elif search_val.startswith('desc:'):
             sort_field = search_val[5:]
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'newfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'oldfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_ASC
 
         return direction, sort_field
 
 
 def search_config(config, prefix='search.'):
     _config = {}
     for key in config.keys():
         if key.startswith(prefix):
             _config[key[len(prefix):]] = config[key]
     return _config
 
 
 def searcher_from_config(config, prefix='search.'):
     _config = search_config(config, prefix)
 
     if 'location' not in _config:
         _config['location'] = default_location
     if 'es_version' not in _config:
-        # use old legacy ES version set to 2
+        # use an old legacy ES version set to 2
         _config['es_version'] = '2'
 
     imported = importlib.import_module(_config.get('module', default_searcher))
     searcher = imported.Searcher(config=_config)
     return searcher

rhodecode/lib/jsonalchemy.py

@@ -1,278 +1,173 b''
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import collections
-
 import sqlalchemy
 from sqlalchemy import UnicodeText
-from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.ext.mutable import Mutable, \
+    MutableList as MutationList, \
+    MutableDict as MutationDict
 
-from rhodecode.lib.ext_json import json
-from rhodecode.lib.utils2 import safe_unicode
+from rhodecode.lib import ext_json
 
 
 class JsonRaw(str):
     """
     Allows interacting with a JSON types field using a raw string.
 
     For example::
         db_instance = JsonTable()
         db_instance.enabled = True
         db_instance.json_data = JsonRaw('{"a": 4}')
 
     This will bypass serialization/checks, and allow storing
     raw values
     """
     pass
 
 
-# Set this to the standard dict if Order is not required
-DictClass = collections.OrderedDict
-
-
 class JSONEncodedObj(sqlalchemy.types.TypeDecorator):
     """
     Represents an immutable structure as a json-encoded string.
 
     If default is, for example, a dict, then a NULL value in the
     database will be exposed as an empty dict.
     """
 
     impl = UnicodeText
     safe = True
-    enforce_unicode = True
+    enforce_str = True
 
     def __init__(self, *args, **kwargs):
         self.default = kwargs.pop('default', None)
         self.safe = kwargs.pop('safe_json', self.safe)
-        self.enforce_unicode = kwargs.pop('enforce_unicode', self.enforce_unicode)
+        self.enforce_str = kwargs.pop('enforce_str', self.enforce_str)
         self.dialect_map = kwargs.pop('dialect_map', {})
         super(JSONEncodedObj, self).__init__(*args, **kwargs)
 
     def load_dialect_impl(self, dialect):
         if dialect.name in self.dialect_map:
             return dialect.type_descriptor(self.dialect_map[dialect.name])
         return dialect.type_descriptor(self.impl)
 
     def process_bind_param(self, value, dialect):
         if isinstance(value, JsonRaw):
             value = value
         elif value is not None:
-            value = json.dumps(value)
-            if self.enforce_unicode:
-                value = safe_unicode(value)
+            if self.enforce_str:
+                value = ext_json.str_json(value)
+            else:
+                value = ext_json.json.dumps(value)
         return value
 
     def process_result_value(self, value, dialect):
         if self.default is not None and (not value or value == '""'):
             return self.default()
 
         if value is not None:
             try:
-                value = json.loads(value, object_pairs_hook=DictClass)
-            except Exception as e:
+                value = ext_json.json.loads(value)
+            except Exception:
                 if self.safe and self.default is not None:
                     return self.default()
                 else:
                     raise
         return value
 
 
 class MutationObj(Mutable):
+
     @classmethod
     def coerce(cls, key, value):
         if isinstance(value, dict) and not isinstance(value, MutationDict):
             return MutationDict.coerce(key, value)
         if isinstance(value, list) and not isinstance(value, MutationList):
             return MutationList.coerce(key, value)
         return value
 
     def de_coerce(self):
         return self
 
     @classmethod
     def _listen_on_attribute(cls, attribute, coerce, parent_cls):
         key = attribute.key
         if parent_cls is not attribute.class_:
             return
 
         # rely on "propagate" here
         parent_cls = attribute.class_
 
         def load(state, *args):
             val = state.dict.get(key, None)
             if coerce:
                 val = cls.coerce(key, val)
                 state.dict[key] = val
             if isinstance(val, cls):
                 val._parents[state.obj()] = key
 
         def set(target, value, oldvalue, initiator):
             if not isinstance(value, cls):
                 value = cls.coerce(key, value)
             if isinstance(value, cls):
                 value._parents[target.obj()] = key
             if isinstance(oldvalue, cls):
                 oldvalue._parents.pop(target.obj(), None)
             return value
 
         def pickle(state, state_dict):
             val = state.dict.get(key, None)
             if isinstance(val, cls):
                 if 'ext.mutable.values' not in state_dict:
                     state_dict['ext.mutable.values'] = []
                 state_dict['ext.mutable.values'].append(val)
 
         def unpickle(state, state_dict):
             if 'ext.mutable.values' in state_dict:
                 for val in state_dict['ext.mutable.values']:
                     val._parents[state.obj()] = key
 
         sqlalchemy.event.listen(parent_cls, 'load', load, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True,
                                 propagate=True)
 
 
-class MutationDict(MutationObj, DictClass):
-    @classmethod
-    def coerce(cls, key, value):
-        """Convert plain dictionary to MutationDict"""
-        self = MutationDict(
-            (k, MutationObj.coerce(key, v)) for (k, v) in value.items())
-        self._key = key
-        return self
-
-    def de_coerce(self):
-        return dict(self)
-
-    def __setitem__(self, key, value):
-        # Due to the way OrderedDict works, this is called during __init__.
-        # At this time we don't have a key set, but what is more, the value
-        # being set has already been coerced. So special case this and skip.
-        if hasattr(self, '_key'):
-            value = MutationObj.coerce(self._key, value)
-        DictClass.__setitem__(self, key, value)
-        self.changed()
-
-    def __delitem__(self, key):
-        DictClass.__delitem__(self, key)
-        self.changed()
-
-    def __setstate__(self, state):
-        self.__dict__ = state
-
-    def __reduce_ex__(self, proto):
-        # support pickling of MutationDicts
-        d = dict(self)
-        return (self.__class__, (d,))
-
-
-class MutationList(MutationObj, list):
-    @classmethod
-    def coerce(cls, key, value):
-        """Convert plain list to MutationList"""
-        self = MutationList((MutationObj.coerce(key, v) for v in value))
-        self._key = key
-        return self
-
-    def de_coerce(self):
-        return list(self)
-
-    def __setitem__(self, idx, value):
-        list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def __setslice__(self, start, stop, values):
-        list.__setslice__(self, start, stop,
-                          (MutationObj.coerce(self._key, v) for v in values))
-        self.changed()
-
-    def __delitem__(self, idx):
-        list.__delitem__(self, idx)
-        self.changed()
-
-    def __delslice__(self, start, stop):
-        list.__delslice__(self, start, stop)
-        self.changed()
-
-    def append(self, value):
-        list.append(self, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def insert(self, idx, value):
-        list.insert(self, idx, MutationObj.coerce(self._key, value))
-        self.changed()
-
-    def extend(self, values):
-        list.extend(self, (MutationObj.coerce(self._key, v) for v in values))
-        self.changed()
-
-    def pop(self, *args, **kw):
-        value = list.pop(self, *args, **kw)
-        self.changed()
-        return value
-
-    def remove(self, value):
-        list.remove(self, value)
-        self.changed()
-
-
 def JsonType(impl=None, **kwargs):
     """
     Helper for using a mutation obj, it allows to use .with_variant easily.
     example::
 
         settings = Column('settings_json',
             MutationObj.as_mutable(
             JsonType(dialect_map=dict(mysql=UnicodeText(16384))))
     """
 
     if impl == 'list':
         return JSONEncodedObj(default=list, **kwargs)
     elif impl == 'dict':
-        return JSONEncodedObj(default=DictClass, **kwargs)
+        return JSONEncodedObj(default=dict, **kwargs)
     else:
         return JSONEncodedObj(**kwargs)
 
 
-JSON = MutationObj.as_mutable(JsonType())
-"""
-A type to encode/decode JSON on the fly
-
-sqltype is the string type for the underlying DB column::
-
-    Column(JSON) (defaults to UnicodeText)
-"""
-
-JSONDict = MutationObj.as_mutable(JsonType('dict'))
-"""
-A type to encode/decode JSON dictionaries on the fly
-"""
-
-JSONList = MutationObj.as_mutable(JsonType('list'))
-"""
-A type to encode/decode JSON lists` on the fly
-"""