python3: fixed various code issues

- iterators
- use of u''

rhodecode/api/__init__.py

@@ -1,578 +1,578 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import sys
 import types
 import fnmatch
 
 import decorator
 import venusian
 from collections import OrderedDict
 
 from pyramid.exceptions import ConfigurationError
 from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPNotFound
 
 from rhodecode.api.exc import (
     JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
 DEFAULT_URL = '/_admin/apiv2'
 
 
 def find_methods(jsonrpc_methods, pattern):
     matches = OrderedDict()
     if not isinstance(pattern, (list, tuple)):
         pattern = [pattern]
 
     for single_pattern in pattern:
         for method_name, method in jsonrpc_methods.items():
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
 
 
 class ExtJsonRenderer(object):
     """
     Custom renderer that mkaes use of our ext_json lib
 
     """
 
     def __init__(self, serializer=json.dumps, **kw):
         """ Any keyword arguments will be passed to the ``serializer``
         function."""
         self.serializer = serializer
         self.kw = kw
 
     def __call__(self, info):
         """ Returns a plain JSON-encoded string with content-type
         ``application/json``. The content-type may be overridden by
         setting ``request.response.content_type``."""
 
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
 
             return self.serializer(value, **self.kw)
 
         return _render
 
 
 def jsonrpc_response(request, result):
     rpc_id = getattr(request, 'rpc_id', None)
     response = request.response
 
     # store content_type before render is called
     ct = response.content_type
 
     ret_value = ''
     if rpc_id:
         ret_value = {
             'id': rpc_id,
             'result': result,
             'error': None,
         }
 
         # fetch deprecation warnings, and store it inside results
         deprecation = getattr(request, 'rpc_deprecation', None)
         if deprecation:
             ret_value['DEPRECATION_WARNING'] = deprecation
 
     raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
     response.body = safe_str(raw_body, response.charset)
 
     if ct == response.default_content_type:
         response.content_type = 'application/json'
 
     return response
 
 
 def jsonrpc_error(request, message, retid=None, code=None, headers=None):
     """
     Generate a Response object with a JSON-RPC error body
 
     :param code:
     :param retid:
     :param message:
     """
     err_dict = {'id': retid, 'result': None, 'error': message}
     body = render(DEFAULT_RENDERER, err_dict, request=request).encode('utf-8')
 
     return Response(
         body=body,
         status=code,
         content_type='application/json',
         headerlist=headers
     )
 
 
 def exception_view(exc, request):
     rpc_id = getattr(request, 'rpc_id', None)
 
     if isinstance(exc, JSONRPCError):
         fault_message = safe_str(exc.message)
         log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCValidationError):
         colander_exc = exc.colander_exception
         # TODO(marcink): think maybe of nicer way to serialize errors ?
         fault_message = colander_exc.asdict()
         log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCForbidden):
         fault_message = 'Access was denied to this resource.'
         log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, HTTPNotFound):
         method = request.rpc_method
         log.debug('json-rpc method `%s` not found in list of '
                   'api calls: %s, rpc_id:%s',
                   method, request.registry.jsonrpc_methods.keys(), rpc_id)
 
         similar = 'none'
         try:
             similar_paterns = ['*{}*'.format(x) for x in method.split('_')]
             similar_found = find_methods(
                 request.registry.jsonrpc_methods, similar_paterns)
             similar = ', '.join(similar_found.keys()) or similar
         except Exception:
             # make the whole above block safe
             pass
 
         fault_message = "No such method: {}. Similar methods: {}".format(
             method, similar)
     else:
         fault_message = 'undefined error'
         exc_info = exc.exc_info()
         store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = "{}.{}".format(exc.__class__.__module__, exc.__class__.__name__)
             statsd.incr('rhodecode_exception_total',
                         tags=["exc_source:api", "type:{}".format(exc_type)])
 
     return jsonrpc_error(request, fault_message, rpc_id)
 
 
 def request_view(request):
     """
     Main request handling method. It handles all logic to call a specific
     exposed method
     """
     # cython compatible inspect
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
         api_user = User.get_by_auth_token(request.rpc_api_key)
 
         if api_user is None:
             return jsonrpc_error(
                 request, retid=request.rpc_id, message='Invalid API KEY')
 
         if not api_user.active:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='Request from this user not allowed')
 
         # check if we are allowed to use this IP
         auth_u = AuthUser(
             api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
         if not auth_u.ip_allowed:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='Request from IP:%s not allowed' % (
                     request.rpc_ip_addr,))
         else:
             log.info('Access for IP:%s allowed', request.rpc_ip_addr)
 
         # register our auth-user
         request.rpc_user = auth_u
         request.environ['rc_auth_user_id'] = auth_u.user_id
 
         # now check if token is valid for API
         auth_token = request.rpc_api_key
         token_match = api_user.authenticate_by_token(
             auth_token, roles=[UserApiKeys.ROLE_API])
         invalid_token = not token_match
 
         log.debug('Checking if API KEY is valid with proper role')
         if invalid_token:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='API KEY invalid or, has bad role for an API call')
 
     except Exception:
         log.exception('Error on API AUTH')
         return jsonrpc_error(
             request, retid=request.rpc_id, message='Invalid API KEY')
 
     method = request.rpc_method
     func = request.registry.jsonrpc_methods[method]
 
     # now that we have a method, add request._req_params to
     # self.kargs and dispatch control to WGIController
     argspec = inspect.getargspec(func)
     arglist = argspec[0]
     defaults = map(type, argspec[3] or [])
     default_empty = types.NotImplementedType
 
     # kw arguments required by this method
-    func_kwargs = dict(itertools.izip_longest(
+    func_kwargs = dict(itertools.zip_longest(
         reversed(arglist), reversed(defaults), fillvalue=default_empty))
 
     # This attribute will need to be first param of a method that uses
     # api_key, which is translated to instance of user at that name
     user_var = 'apiuser'
     request_var = 'request'
 
     for arg in [user_var, request_var]:
         if arg not in arglist:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message='This method [%s] does not support '
                         'required parameter `%s`' % (func.__name__, arg))
 
     # get our arglist and check if we provided them as args
     for arg, default in func_kwargs.items():
         if arg in [user_var, request_var]:
             # user_var and request_var are pre-hardcoded parameters and we
             # don't need to do any translation
             continue
 
         # skip the required param check if it's default value is
         # NotImplementedType (default_empty)
         if default == default_empty and arg not in request.rpc_params:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message=('Missing non optional `%s` arg in JSON DATA' % arg)
             )
 
     # sanitize extra passed arguments
     for k in request.rpc_params.keys()[:]:
         if k not in func_kwargs:
             del request.rpc_params[k]
 
     call_params = request.rpc_params
     call_params.update({
         'request': request,
         'apiuser': auth_u
     })
 
     # register some common functions for usage
     attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
 
     statsd = request.registry.statsd
 
     try:
         ret_value = func(**call_params)
         resp = jsonrpc_response(request, ret_value)
         if statsd:
             statsd.incr('rhodecode_api_call_success_total')
         return resp
     except JSONRPCBaseError:
         raise
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
         exc_id, exc_type_name = store_exception(
             id(exc_info), exc_info, prefix='rhodecode-api')
         error_headers = [('RhodeCode-Exception-Id', str(exc_id)),
                          ('RhodeCode-Exception-Type', str(exc_type_name))]
         err_resp = jsonrpc_error(
             request, retid=request.rpc_id, message='Internal server error',
             headers=error_headers)
         if statsd:
             statsd.incr('rhodecode_api_call_fail_total')
         return err_resp
 
 
 def setup_request(request):
     """
     Parse a JSON-RPC request body. It's used inside the predicates method
     to validate and bootstrap requests for usage in rpc calls.
 
     We need to raise JSONRPCError here if we want to return some errors back to
     user.
     """
 
     log.debug('Executing setup request: %r', request)
     request.rpc_ip_addr = get_ip_addr(request.environ)
     # TODO(marcink): deprecate GET at some point
     if request.method not in ['POST', 'GET']:
         log.debug('unsupported request method "%s"', request.method)
         raise JSONRPCError(
             'unsupported request method "%s". Please use POST' % request.method)
 
     if 'CONTENT_LENGTH' not in request.environ:
         log.debug("No Content-Length")
         raise JSONRPCError("Empty body, No Content-Length in request")
 
     else:
         length = request.environ['CONTENT_LENGTH']
         log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
     log.debug("Loading JSON body now")
     try:
         json_body = json.loads(raw_body)
     except ValueError as e:
         # catch JSON errors Here
         raise JSONRPCError("JSON parse error ERR:%s RAW:%r" % (e, raw_body))
 
     request.rpc_id = json_body.get('id')
     request.rpc_method = json_body.get('method')
 
     # check required base parameters
     try:
         api_key = json_body.get('api_key')
         if not api_key:
             api_key = json_body.get('auth_token')
 
         if not api_key:
             raise KeyError('api_key or auth_token')
 
         # TODO(marcink): support passing in token in request header
 
         request.rpc_api_key = api_key
         request.rpc_id = json_body['id']
         request.rpc_method = json_body['method']
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
         log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError('Incorrect JSON data. Missing %s' % e)
 
     log.debug('setup complete, now handling method:%s rpcid:%s',
               request.rpc_method, request.rpc_id, )
 
 
 class RoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'jsonrpc route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if self.val:
             # potentially setup and bootstrap our call
             setup_request(request)
 
             # Always return True so that even if it isn't a valid RPC it
             # will fall through to the underlaying handlers like notfound_view
             return True
 
 
 class NotFoundPredicate(object):
     def __init__(self, val, config):
         self.val = val
         self.methods = config.registry.jsonrpc_methods
 
     def text(self):
         return 'jsonrpc method not found = {}.'.format(self.val)
 
     phash = text
 
     def __call__(self, info, request):
         return hasattr(request, 'rpc_method')
 
 
 class MethodPredicate(object):
     def __init__(self, val, config):
         self.method = val
 
     def text(self):
         return 'jsonrpc method = %s' % self.method
 
     phash = text
 
     def __call__(self, context, request):
         # we need to explicitly return False here, so pyramid doesn't try to
         # execute our view directly. We need our main handler to execute things
         return getattr(request, 'rpc_method') == self.method
 
 
 def add_jsonrpc_method(config, view, **kwargs):
     # pop the method name
     method = kwargs.pop('method', None)
 
     if method is None:
         raise ConfigurationError(
             'Cannot register a JSON-RPC method without specifying the "method"')
 
     # we define custom predicate, to enable to detect conflicting methods,
     # those predicates are kind of "translation" from the decorator variables
     # to internal predicates names
 
     kwargs['jsonrpc_method'] = method
 
     # register our view into global view store for validation
     config.registry.jsonrpc_methods[method] = view
 
     # we're using our main request_view handler, here, so each method
     # has a unified handler for itself
     config.add_view(request_view, route_name='apiv2', **kwargs)
 
 
 class jsonrpc_method(object):
     """
     decorator that works similar to @add_view_config decorator,
     but tailored for our JSON RPC
     """
 
     venusian = venusian  # for testing injection
 
     def __init__(self, method=None, **kwargs):
         self.method = method
         self.kwargs = kwargs
 
     def __call__(self, wrapped):
         kwargs = self.kwargs.copy()
         kwargs['method'] = self.method or wrapped.__name__
         depth = kwargs.pop('_depth', 0)
 
         def callback(context, name, ob):
             config = context.config.with_package(info.module)
             config.add_jsonrpc_method(view=ob, **kwargs)
 
         info = venusian.attach(wrapped, callback, category='pyramid',
                                depth=depth + 1)
         if info.scope == 'class':
             # ensure that attr is set if decorating a class method
             kwargs.setdefault('attr', wrapped.__name__)
 
         kwargs['_info'] = info.codeinfo  # fbo action_method
         return wrapped
 
 
 class jsonrpc_deprecated_method(object):
     """
     Marks method as deprecated, adds log.warning, and inject special key to
     the request variable to mark method as deprecated.
     Also injects special docstring that extract_docs will catch to mark
     method as deprecated.
 
     :param use_method: specify which method should be used instead of
         the decorated one
 
     Use like::
 
         @jsonrpc_method()
         @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
         def old_func(request, apiuser, arg1, arg2):
             ...
     """
 
     def __init__(self, use_method, deprecated_at_version):
         self.use_method = use_method
         self.deprecated_at_version = deprecated_at_version
         self.deprecated_msg = ''
 
     def __call__(self, func):
         self.deprecated_msg = 'Please use method `{method}` instead.'.format(
             method=self.use_method)
 
         docstring = """\n
         .. deprecated:: {version}
 
            {deprecation_message}
 
         {original_docstring}
         """
         func.__doc__ = docstring.format(
             version=self.deprecated_at_version,
             deprecation_message=self.deprecated_msg,
             original_docstring=func.__doc__)
         return decorator.decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         log.warning('DEPRECATED API CALL on function %s, please '
                     'use `%s` instead', func, self.use_method)
         # alter function docstring to mark as deprecated, this is picked up
         # via fabric file that generates API DOC.
         result = func(*fargs, **fkwargs)
 
         request = fargs[0]
         request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
         return result
 
 
 def add_api_methods(config):
     from rhodecode.api.views import (
         deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
         server_api, search_api, testing_api, user_api, user_group_api)
 
     config.scan('rhodecode.api.views')
 
 
 def includeme(config):
     plugin_module = 'rhodecode.api'
     plugin_settings = get_plugin_settings(
         plugin_module, config.registry.settings)
 
     if not hasattr(config.registry, 'jsonrpc_methods'):
         config.registry.jsonrpc_methods = OrderedDict()
 
     # match filter by given method only
     config.add_view_predicate('jsonrpc_method', MethodPredicate)
     config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
 
     config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer(
         serializer=json.dumps, indent=4))
     config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
 
     config.add_route_predicate(
         'jsonrpc_call', RoutePredicate)
 
     config.add_route(
         'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
 
     # register some exception handling view
     config.add_view(exception_view, context=JSONRPCBaseError)
     config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
 
     add_api_methods(config)

rhodecode/api/views/server_api.py

@@ -1,419 +1,419 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import itertools
 import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_repo_store(request, apiuser):
     """
     Returns the |RCE| repository storage information.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     path = VcsSettingsModel().get_repos_location()
     return {"path": path}
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 
 @jsonrpc_method()
 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
     """
     Triggers a session cleanup action.
 
     If the ``older_then`` option is set, only sessions that hasn't been
     accessed in the given number of days will be removed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param older_then: Deletes session that hasn't been accessed
         in given number of days.
     :type older_then: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result: {
         "backend": "<type of backend>",
         "sessions_removed": <number_of_removed_sessions>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during session cleanup'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     older_then = safe_int(Optional.extract(older_then)) or 60
     older_than_seconds = 60 * 60 * 24 * older_then
 
     config = system_info.rhodecode_config().get_value()['value']['config']
     session_model = user_sessions.get_session_handler(
         config.get('beaker.session.type', 'memory'))(config)
 
     backend = session_model.SESSION_TYPE
     try:
         cleaned = session_model.clean_sessions(
             older_than_seconds=older_than_seconds)
         return {'sessions_removed': cleaned, 'backend': backend}
     except user_sessions.CleanupCommand as msg:
         return {'cleanup_command': msg.message, 'backend': backend}
     except Exception as e:
         log.exception('Failed session cleanup')
         raise JSONRPCError(
             'Error occurred during session cleanup'
         )
 
 
 @jsonrpc_method()
 def get_method(request, apiuser, pattern=Optional('*')):
     """
     Returns list of all available API methods. By default match pattern
     os "*" but any other pattern can be specified. eg *comment* will return
     all methods with comment inside them. If just single method is matched
     returned data will also include method specification
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param pattern: pattern to match method names against
     :type pattern: Optional("*")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "changeset_comment",
         "comment_pull_request",
         "comment_commit"
       ]
       error :  null
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "comment_commit",
         {
           "apiuser": "<RequiredType>",
           "comment_type": "<Optional:u'note'>",
           "commit_id": "<RequiredType>",
           "message": "<RequiredType>",
           "repoid": "<RequiredType>",
           "request": "<RequiredType>",
           "resolves_comment_id": "<Optional:None>",
           "status": "<Optional:None>",
           "userid": "<Optional:<OptionalAttr:apiuser>>"
         }
       ]
       error :  null
     """
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     pattern = Optional.extract(pattern)
 
     matches = find_methods(request.registry.jsonrpc_methods, pattern)
 
     args_desc = []
     if len(matches) == 1:
         func = matches[matches.keys()[0]]
 
         argspec = inspect.getargspec(func)
         arglist = argspec[0]
         defaults = map(repr, argspec[3] or [])
 
         default_empty = '<RequiredType>'
 
         # kw arguments required by this method
-        func_kwargs = dict(itertools.izip_longest(
+        func_kwargs = dict(itertools.zip_longest(
             reversed(arglist), reversed(defaults), fillvalue=default_empty))
         args_desc.append(func_kwargs)
 
     return matches.keys() + args_desc
 
 
 @jsonrpc_method()
 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
     """
     Stores sent exception inside the built-in exception tracker in |RCE| server.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     :param exc_data_json: JSON data with exception e.g
         {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
     :type exc_data_json: JSON data
 
     :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
     :type prefix: Optional("rhodecode")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": {
         "exc_id": 139718459226384,
         "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
       }
       error :  null
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     prefix = Optional.extract(prefix)
     exc_id = exc_tracking.generate_id()
 
     try:
         exc_data = json.loads(exc_data_json)
     except Exception:
         log.error('Failed to parse JSON: %r', exc_data_json)
         raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
                            'Please make sure it contains a valid JSON.')
 
     try:
         exc_traceback = exc_data['exc_traceback']
         exc_type_name = exc_data['exc_type_name']
     except KeyError as err:
         raise JSONRPCError('Missing exc_traceback, or exc_type_name '
                            'in exc_data_json field. Missing: {}'.format(err))
 
     exc_tracking._store_exception(
         exc_id=exc_id, exc_traceback=exc_traceback,
         exc_type_name=exc_type_name, prefix=prefix)
 
     exc_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=exc_id)
     return {'exc_id': exc_id, 'exc_url': exc_url}

rhodecode/apps/admin/views/permissions.py

@@ -1,479 +1,479 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import logging
 import formencode
 import formencode.htmlfill
 import datetime
 from pyramid.interfaces import IRoutesMapper
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
 from rhodecode import events
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.utils2 import aslist, safe_unicode
 from rhodecode.model.db import (
     or_, coalesce, User, UserIpMap, UserSshKeys)
 from rhodecode.model.forms import (
     ApplicationPermissionsForm, ObjectPermissionsForm, UserPermissionsForm)
 from rhodecode.model.meta import Session
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminPermissionsView(BaseAppView, DataGridAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=self.request.translate)
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_application(self):
         c = self.load_default_context()
         c.active = 'application'
 
         c.user = User.get_default_user(refresh=True)
 
         app_settings = c.rc_config
 
         defaults = {
             'anonymous': c.user.active,
             'default_register_message': app_settings.get(
                 'rhodecode_register_message')
         }
         defaults.update(c.user.get_default_perms())
 
         data = render('rhodecode:templates/admin/permissions/permissions.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_application_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'application'
 
         _form = ApplicationPermissionsForm(
             self.request.translate,
             [x[0] for x in c.register_choices],
             [x[0] for x in c.password_reset_choices],
             [x[0] for x in c.extern_activate_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_application_permissions(form_result)
 
             settings = [
                 ('register_message', 'default_register_message'),
             ]
             for setting, form_key in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key])
                 Session().add(sett)
 
             Session().commit()
             h.flash(_('Application permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_application'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_objects(self):
         c = self.load_default_context()
         c.active = 'objects'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_objects_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'objects'
 
         _form = ObjectPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_perms_choices],
             [x[0] for x in c.group_perms_choices],
             [x[0] for x in c.user_group_perms_choices],
         )()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_object_permissions(form_result)
 
             Session().commit()
             h.flash(_('Object permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_object'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_branch(self):
         c = self.load_default_context()
         c.active = 'branch'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_global(self):
         c = self.load_default_context()
         c.active = 'global'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
 
         _form = UserPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_create_choices],
             [x[0] for x in c.repo_create_on_write_choices],
             [x[0] for x in c.repo_group_create_choices],
             [x[0] for x in c.user_group_create_choices],
             [x[0] for x in c.fork_choices],
             [x[0] for x in c.inherit_default_permission_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_user_permissions(form_result)
 
             Session().commit()
             h.flash(_('Global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_ips(self):
         c = self.load_default_context()
         c.active = 'ips'
 
         c.user = User.get_default_user(refresh=True)
         c.user_ip_map = (
             UserIpMap.query().filter(UserIpMap.user == c.user).all())
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_overview(self):
         c = self.load_default_context()
         c.active = 'perms'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_token_access(self):
         from rhodecode import CONFIG
 
         c = self.load_default_context()
         c.active = 'auth_token_access'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
 
         mapper = self.request.registry.queryUtility(IRoutesMapper)
         c.view_data = []
 
-        _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
+        _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
         introspector = self.request.registry.introspector
 
         view_intr = {}
         for view_data in introspector.get_category('views'):
             intr = view_data['introspectable']
 
             if 'route_name' in intr and intr['attr']:
                 view_intr[intr['route_name']] = '{}:{}'.format(
                     str(intr['derived_callable'].__name__), intr['attr']
                 )
 
         c.whitelist_key = 'api_access_controllers_whitelist'
         c.whitelist_file = CONFIG.get('__file__')
         whitelist_views = aslist(
             CONFIG.get(c.whitelist_key), sep=',')
 
         for route_info in mapper.get_routes():
             if not route_info.name.startswith('__'):
                 routepath = route_info.pattern
 
                 def replace(matchobj):
                     if matchobj.group(1):
                         return "{%s}" % matchobj.group(1).split(':')[0]
                     else:
                         return "{%s}" % matchobj.group(2)
 
                 routepath = _argument_prog.sub(replace, routepath)
 
                 if not routepath.startswith('/'):
                     routepath = '/' + routepath
 
                 view_fqn = view_intr.get(route_info.name, 'NOT AVAILABLE')
                 active = view_fqn in whitelist_views
                 c.view_data.append((route_info.name, view_fqn, routepath, active))
 
         c.whitelist_views = whitelist_views
         return self._get_template_context(c)
 
     def ssh_enabled(self):
         return self.request.registry.settings.get(
             'ssh.generate_authorized_keyfile')
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys(self):
         c = self.load_default_context()
         c.active = 'ssh_keys'
         c.ssh_enabled = self.ssh_enabled()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys_data(self):
         _ = self.request.translate
         self.load_default_context()
         column_map = {
             'fingerprint': 'ssh_key_fingerprint',
             'username': User.username
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         ssh_keys_data_total_count = UserSshKeys.query()\
             .count()
 
         # json generate
         base_q = UserSshKeys.query().join(UserSshKeys.user)
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 UserSshKeys.ssh_key_fingerprint.ilike(like_expression),
             ))
 
         users_data_total_filtered_count = base_q.count()
 
         sort_col = self._get_order_col(order_by, UserSshKeys)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         ssh_keys = base_q.all()
 
         ssh_keys_data = []
         for ssh_key in ssh_keys:
             ssh_keys_data.append({
                 "username": h.gravatar_with_user(self.request, ssh_key.user.username),
                 "fingerprint": ssh_key.ssh_key_fingerprint,
                 "description": ssh_key.description,
                 "created_on": h.format_date(ssh_key.created_on),
                 "accessed_on": h.format_date(ssh_key.accessed_on),
                 "action": h.link_to(
                     _('Edit'), h.route_path('edit_user_ssh_keys',
                                             user_id=ssh_key.user.user_id))
             })
 
         data = ({
             'draw': draw,
             'data': ssh_keys_data,
             'recordsTotal': ssh_keys_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
         })
 
         return data
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_update(self):
         _ = self.request.translate
         self.load_default_context()
 
         ssh_enabled = self.ssh_enabled()
         key_file = self.request.registry.settings.get(
             'ssh.authorized_keys_file_path')
         if ssh_enabled:
             events.trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_('Updated SSH keys file: {}').format(key_file),
                     category='success')
         else:
             h.flash(_('SSH key support is disabled in .ini file'),
                     category='warning')
 
         raise HTTPFound(h.route_path('admin_permissions_ssh_keys'))

rhodecode/apps/file_store/utils.py

@@ -1,58 +1,57 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-
+import io
 import uuid
-from io import StringIO
 import pathlib2
 
 
 def get_file_storage(settings):
     from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
     from rhodecode.apps.file_store import config_keys
     store_path = settings.get(config_keys.store_path)
     return LocalFileStorage(base_path=store_path)
 
 
 def splitext(filename):
     ext = ''.join(pathlib2.Path(filename).suffixes)
     return filename, ext
 
 
 def uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
 
     :param filename: the original filename
     :param randomized: define if filename should be stable (sha1 based) or randomized
     """
 
     _, ext = splitext(filename)
     if randomized:
         uid = uuid.uuid4()
     else:
         hash_key = '{}.{}'.format(filename, 'store')
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
 
 
 def bytes_to_file_obj(bytes_data):
-    return StringIO.StringIO(bytes_data)
+    return io.StringIO(bytes_data)

rhodecode/apps/login/tests/test_login.py

@@ -1,580 +1,580 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urllib.parse
 
 import mock
 import pytest
 
 from rhodecode.tests import (
     assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
     no_newline_id_generator)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.lib.auth import check_password
 from rhodecode.lib import helpers as h
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import User, Notification, UserApiKeys
 from rhodecode.model.meta import Session
 
 fixture = Fixture()
 
 whitelist_view = ['RepoCommitsView:repo_commit_raw']
 
 
 def route_path(name, params=None, **kwargs):
     import urllib.request, urllib.parse, urllib.error
     from rhodecode.apps._base import ADMIN_PREFIX
 
     base_url = {
         'login': ADMIN_PREFIX + '/login',
         'logout': ADMIN_PREFIX + '/logout',
         'register': ADMIN_PREFIX + '/register',
         'reset_password':
             ADMIN_PREFIX + '/password_reset',
         'reset_password_confirmation':
             ADMIN_PREFIX + '/password_reset_confirmation',
 
         'admin_permissions_application':
             ADMIN_PREFIX + '/permissions/application',
         'admin_permissions_application_update':
             ADMIN_PREFIX + '/permissions/application/update',
 
         'repo_commit_raw': '/{repo_name}/raw-changeset/{commit_id}'
 
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.parse.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app')
 class TestLoginController(object):
     destroy_users = set()
 
     @classmethod
     def teardown_class(cls):
         fixture.destroy_users(cls.destroy_users)
 
     def teardown_method(self, method):
         for n in Notification.query().all():
             Session().delete(n)
 
         Session().commit()
         assert Notification.query().all() == []
 
     def test_index(self):
         response = self.app.get(route_path('login'))
         assert response.status == '200 OK'
         # Test response...
 
     def test_login_admin_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'test12'}, status=302)
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_admin'
         response.mustcontain('logout')
 
     def test_login_regular_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_regular',
                                   'password': 'test12'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_regular'
         response.mustcontain('logout')
 
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.auth_restriction(self.app._pyramid_registry,
                                       RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_regular_forbidden_when_scope_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.scope_restriction(self.app._pyramid_registry,
                                        RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_ok_came_from(self):
         test_came_from = '/_admin/users?branch=stable'
         _url = '{}?came_from={}'.format(route_path('login'), test_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
 
         assert 'branch=stable' in response.location
         response = response.follow()
 
         assert response.status == '200 OK'
         response.mustcontain('Users administration')
 
     def test_redirect_to_login_with_get_args(self):
         with fixture.anon_access(False):
             kwargs = {'branch': 'stable'}
             response = self.app.get(
                 h.route_path('repo_summary', repo_name=HG_REPO, _query=kwargs),
                 status=302)
 
-            response_query = urllib.parse.urlparse.parse_qsl(response.location)
+            response_query = urllib.parse.parse_qsl(response.location)
             assert 'branch=stable' in response_query[0][1]
 
     def test_login_form_with_get_args(self):
         _url = '{}?came_from=/_admin/users,branch=stable'.format(route_path('login'))
         response = self.app.get(_url)
         assert 'branch%3Dstable' in response.form.action
 
     @pytest.mark.parametrize("url_came_from", [
         'data:text/html,<script>window.alert("xss")</script>',
         'mailto:[e-mail]',
         'file:///etc/passwd',
         'ftp://some.ftp.server',
         'http://other.domain',
         '/\r\nX-Forwarded-Host: http://example.org',
     ], ids=no_newline_id_generator)
     def test_login_bad_came_froms(self, url_came_from):
         _url = '{}?came_from={}'.format(route_path('login'), url_came_from)
         response = self.app.post(
             _url,
             {'username': 'test_admin', 'password': 'test12'})
         assert response.status == '302 Found'
         response = response.follow()
         assert response.status == '200 OK'
         assert response.request.path == '/'
 
     def test_login_short_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'as'})
         assert response.status == '200 OK'
 
         response.mustcontain('Enter 3 characters or more')
 
     def test_login_wrong_non_ascii_password(self, user_regular):
         response = self.app.post(
             route_path('login'),
             {'username': user_regular.username,
              'password': u'invalid-non-asci\xe4'.encode('utf8')})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_with_non_ascii_password(self, user_util):
         password = u'valid-non-ascii\xe4'
         user = user_util.create_user(password=password)
         response = self.app.post(
             route_path('login'),
             {'username': user.username,
              'password': password})
         assert response.status_code == 302
 
     def test_login_wrong_username_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'error',
                                   'password': 'test12'})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_admin_ok_password_migration(self, real_crypto_backend):
         from rhodecode.lib import auth
 
         # create new user, with sha256 password
         temp_user = 'test_admin_sha256'
         user = fixture.create_user(temp_user)
         user.password = auth._RhodeCodeCryptoSha256().hash_create(
             b'test123')
         Session().add(user)
         Session().commit()
         self.destroy_users.add(temp_user)
         response = self.app.post(route_path('login'),
                                  {'username': temp_user,
                                   'password': 'test123'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == temp_user
         response.mustcontain('logout')
 
         # new password should be bcrypted, after log-in and transfer
         user = User.get_by_username(temp_user)
         assert user.password.startswith('$')
 
     # REGISTRATIONS
     def test_register(self):
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
     def test_register_err_same_username(self):
         uname = 'test_admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': uname,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = 'Username "%(username)s" already exists'
         msg = msg % {'username': uname}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_err_same_email(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_0',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_same_email_case_sensitive(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_1',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_wrong_data(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': 'test',
                 'password_confirmation': 'test',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assert response.status == '200 OK'
         response.mustcontain('An email address must contain a single @')
         response.mustcontain('Enter a value 6 characters long or more')
 
     def test_register_err_username(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'error user',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         response.mustcontain('An email address must contain a single @')
         response.mustcontain(
             'Username may only contain '
             'alphanumeric characters underscores, '
             'periods or dashes and must begin with '
             'alphanumeric character')
 
     def test_register_err_case_sensitive(self):
         usr = 'Test_Admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': usr,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'Username "%(username)s" already exists'
         msg = msg % {'username': usr}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_special_chars(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xxxaxn',
                 'password': 'ąćźżąśśśś',
                 'password_confirmation': 'ąćźżąśśśś',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         msg = u'Invalid characters (non-ascii) in password'
         response.mustcontain(msg)
 
     def test_register_password_mismatch(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': '123qwe',
                 'password_confirmation': 'qwe123',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         msg = u'Passwords do not match'
         response.mustcontain(msg)
 
     def test_register_ok(self):
         username = 'test_regular4'
         password = 'qweqwe'
         email = '[e-mail]'
         name = 'testname'
         lastname = 'testlastname'
 
         # this initializes a session
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
 
         response = self.app.post(
             route_path('register'),
             {
                 'username': username,
                 'password': password,
                 'password_confirmation': password,
                 'email': email,
                 'firstname': name,
                 'lastname': lastname,
                 'admin': True
             },
             status=302
         )  # This should be overridden
 
         assert_session_flash(
             response, 'You have successfully registered with RhodeCode. You can log-in now.')
 
         ret = Session().query(User).filter(
             User.username == 'test_regular4').one()
         assert ret.username == username
         assert check_password(password, ret.password)
         assert ret.email == email
         assert ret.name == name
         assert ret.lastname == lastname
         assert ret.auth_tokens is not None
         assert not ret.admin
 
     def test_forgot_password_wrong_mail(self):
         bad_email = '[e-mail]'
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         response = self.app.post(
             route_path('reset_password'), {'email': bad_email, }
         )
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
     def test_forgot_password(self, user_util):
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         user = user_util.create_user()
         user_id = user.user_id
         email = user.email
 
         response = self.app.post(route_path('reset_password'), {'email': email, })
 
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
         # BAD KEY
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), 'badkey')
         response = self.app.get(confirm_url, status=302)
         assert response.location.endswith(route_path('reset_password'))
         assert_session_flash(response, 'Given reset token is invalid')
 
         response.follow()  # cleanup flash
 
         # GOOD KEY
         key = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == user_id)\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_PASSWORD_RESET)\
             .first()
 
         assert key
 
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), key.api_key)
         response = self.app.get(confirm_url)
         assert response.status == '302 Found'
         assert response.location.endswith(route_path('login'))
 
         assert_session_flash(
             response,
             'Your password reset was successful, '
             'a new password has been sent to your email')
 
         response.follow()
 
     def _get_api_whitelist(self, values=None):
         config = {'api_access_controllers_whitelist': values or []}
         return config
 
     @pytest.mark.parametrize("test_name, auth_token", [
         ('none', None),
         ('empty_string', ''),
         ('fake_number', '123456'),
         ('proper_auth_token', None)
     ])
     def test_access_not_whitelisted_page_via_auth_token(
             self, test_name, auth_token, user_admin):
 
         whitelist = self._get_api_whitelist([])
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert [] == whitelist['api_access_controllers_whitelist']
             if test_name == 'proper_auth_token':
                 # use builtin if api_key is None
                 auth_token = user_admin.api_key
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=302)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('none', None, 302),
         ('empty_string', '', 302),
         ('fake_number', '123456', 302),
         ('proper_auth_token', None, 200)
     ])
     def test_access_whitelisted_page_via_auth_token(
             self, test_name, auth_token, code, user_admin):
 
         whitelist = self._get_api_whitelist(whitelist_view)
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == whitelist['api_access_controllers_whitelist']
 
             if test_name == 'proper_auth_token':
                 auth_token = user_admin.api_key
                 assert auth_token
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('proper_auth_token', None, 200),
         ('wrong_auth_token', '123456', 302),
     ])
     def test_access_whitelisted_page_via_auth_token_bound_to_token(
             self, test_name, auth_token, code, user_admin):
 
         expected_token = auth_token
         if test_name == 'proper_auth_token':
             auth_token = user_admin.api_key
             expected_token = auth_token
             assert auth_token
 
         whitelist = self._get_api_whitelist([
             'RepoCommitsView:repo_commit_raw@{}'.format(expected_token)])
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     def test_access_page_via_extra_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=200)
 
     def test_access_page_via_expired_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             # patch the api key and make it expired
             new_auth_token.expires = 0
             Session().add(new_auth_token)
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=302)

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1877 +1,1877 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest, HTTPConflict)
 
 from pyramid.renderers import render
 
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.exceptions import CommentVersionMismatch
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry
 from rhodecode.lib.vcs.backends.base import (
     EmptyCommit, UpdateFailureReason, unicode_to_reference)
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     func, false, or_, PullRequest, ChangesetComment, ChangesetStatus, Repository,
     PullRequestReviewers)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
         # backward compat., we use for OLD PRs a plain renderer
         c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, search_q=search_q, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments_count = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr,
                 include_drafts=False, count_only=True)
 
             review_statuses = pr.reviewers_statuses(user=self._rhodecode_db_user)
             my_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
             if review_statuses and review_statuses[4]:
                 _review_obj, _user, _reasons, _mandatory, statuses = review_statuses
                 my_review_status = statuses[0][1].status
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name,
                                 short=True),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'my_status': _render('pullrequest_status',
                                      my_review_status),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on),
                                       pr.versions_count),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', comments_count),
                 'comments_raw': comments_count,
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      ancestor_commit,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff, hide_whitespace_changes, diff_context, use_ancestor=True):
 
         target_commit_final = target_commit
         source_commit_final = source_commit
 
         if use_ancestor:
             # we might want to not use it for versions
             target_ref_id = ancestor_commit.raw_id
             target_commit_final = ancestor_commit
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit_final),
             target_node_getter=codeblocks.diffset_node_getter(source_commit_final),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_ref_id, source_ref_id)
 
         return diffset
 
     def _get_range_diffset(self, source_scm, source_repo,
                            commit1, commit2, diff_limit, file_limit,
                            fulldiff, hide_whitespace_changes, diff_context):
         vcs_diff = source_scm.get_diff(
             commit1, commit2,
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
             source_node_getter=codeblocks.diffset_node_getter(commit1),
             target_node_getter=codeblocks.diffset_node_getter(commit2))
 
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
         return diffset
 
     def register_comments_vars(self, c, pull_request, versions, include_drafts=True):
         comments_model = CommentsModel()
 
         # GENERAL COMMENTS with versions #
         q = comments_model._all_general_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         general_comments = q
 
         # pick comments we want to render at current version
         c.comment_versions = comments_model.aggregate_comments(
             general_comments, versions, c.at_version_num)
 
         # INLINE COMMENTS with versions  #
         q = comments_model._all_inline_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         inline_comments = q
 
         c.inline_versions = comments_model.aggregate_comments(
             inline_comments, versions, c.at_version_num, inline=True)
 
         # Comments inline+general
         if c.at_version:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['display']
             c.comments = c.comment_versions[c.at_version_num]['display']
         else:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['until']
             c.comments = c.comment_versions[c.at_version_num]['until']
 
         return general_comments, inline_comments
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_show(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         c.state_progressing = pull_request.is_state_changing()
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         _new_state = {
             'created': PullRequest.STATE_CREATED,
         }.get(self.request.GET.get('force_state'))
         can_force_state = c.is_super_admin or HasRepoPermissionAny('repository.admin')(c.repo_name)
 
         if can_force_state and _new_state:
             with pull_request.set_state(PullRequest.STATE_UPDATING, final_state=_new_state):
                 h.flash(
                     _('Pull Request state was force changed to `{}`').format(_new_state),
                     category='success')
                 Session().commit()
 
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
         force_refresh = str2bool(self.request.GET.get('force_refresh'))
         c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         # fetch global flags of ignore ws or context lines
         diff_context = diffs.get_diff_context(self.request)
         hide_whitespace_changes = diffs.get_diff_whitespace_flag(self.request)
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
 
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
             # not allow to browse versions for closed PR
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
 
         c.commit_versions = PullRequestModel().pr_commits_versions(versions)
 
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.at_version_index = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
                               if prev_at_version and prev_at_version != PullRequest.LATEST_VER
                               else None)
         c.from_version_index = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
             log.warning('Mismatch between the current repo: %s, and target %s',
                         self.db_repo_name, pull_request_at_ver.target_repo.repo_name)
             raise HTTPNotFound()
 
         c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
         c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
         # inject latest version
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         if compare or (at_version and not at_version == PullRequest.LATEST_VER):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
                 c.forbid_adding_reviewers = rules.get('forbid_adding_reviewers')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, auth_user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
 
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
         c.pr_merge_source_commit = _merge_check.source_commit
         c.pr_merge_target_commit = _merge_check.target_commit
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
         c.reviewers_count = pull_request.reviewers_count
         c.observers_count = pull_request.observers_count
 
         # reviewers and statuses
         c.pull_request_default_reviewers_data_json = json.dumps(pull_request.reviewer_data)
         c.pull_request_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
         c.pull_request_set_observers_data_json = collections.OrderedDict({'observers': []})
 
         for review_obj, member, reasons, mandatory, status in pull_request_at_ver.reviewers_statuses():
             member_reviewer = h.reviewer_as_json(
                 member, reasons=reasons, mandatory=mandatory,
                 role=review_obj.role,
                 user_group=review_obj.rule_user_group_data()
             )
 
             current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
             member_reviewer['review_status'] = current_review_status
             member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
             member_reviewer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
         c.pull_request_set_reviewers_data_json = json.dumps(c.pull_request_set_reviewers_data_json)
 
         for observer_obj, member in pull_request_at_ver.observers():
             member_observer = h.reviewer_as_json(
                 member, reasons=[], mandatory=False,
                 role=observer_obj.role,
                 user_group=observer_obj.rule_user_group_data()
             )
             member_observer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_observers_data_json['observers'].append(member_observer)
 
         c.pull_request_set_observers_data_json = json.dumps(c.pull_request_set_observers_data_json)
 
         general_comments, inline_comments = \
             self.register_comments_vars(c, pull_request_latest, versions)
 
         # TODOs
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request_latest)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request_latest)
 
         # Drafts
         c.draft_comments = CommentsModel().get_pull_request_drafts(
             self._rhodecode_db_user.user_id,
             pull_request_latest)
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm
         if shadow_scm:
             commits_source_repo = shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
         version_normalized = version or PullRequest.LATEST_VER
         from_version_normalized = from_version or PullRequest.LATEST_VER
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = self.get_recache_flag()
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
 
         if not force_recache and not c.range_diff_on and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             # NOTE(marcink): we reach potentially unreachable errors when a PR has
             # merge errors resulting in potentially hidden commits in the shadow repo.
             maybe_unreachable = _merge_check.MERGE_CHECK in _merge_check.error_details \
                                 and _merge_check.merge_response
             maybe_unreachable = maybe_unreachable \
                                 and _merge_check.merge_response.metadata.get('unresolved_files')
             log.debug("Using unreachable commits due to MERGE_CHECK in merge simulation")
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm,
                 maybe_unreachable=maybe_unreachable)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             use_ancestor = True
             if from_version_normalized != version_normalized:
                 use_ancestor = False
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
             else:
                 try:
                     c.diffset = self._get_diffset(
                         c.source_repo.repo_name, commits_source_repo,
                         c.ancestor_commit,
                         source_ref_id, target_ref_id,
                         target_commit, source_commit,
                         diff_limit, file_limit, c.fulldiff,
                         hide_whitespace_changes, diff_context,
                         use_ancestor=use_ancestor
                     )
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, c.diffset, diff_commit_cache)
                 except CommitDoesNotExistError:
                     log.exception('Failed to generate diffset')
                     c.missing_commits = True
 
             if not c.missing_commits:
 
                 c.limited_diff = c.diffset.limited_diff
 
                 # calculate removed files that are bound to comments
                 comment_deleted_files = [
                     fname for fname in display_inline_comments
                     if fname not in c.diffset.file_stats]
 
                 c.deleted_files_comments = collections.defaultdict(dict)
                 for fname, per_line_comments in display_inline_comments.items():
                     if fname in comment_deleted_files:
                         c.deleted_files_comments[fname]['stats'] = 0
                         c.deleted_files_comments[fname]['comments'] = list()
                         for lno, comments in per_line_comments.items():
                             c.deleted_files_comments[fname]['comments'].extend(comments)
 
                 # maybe calculate the range diff
                 if c.range_diff_on:
                     # TODO(marcink): set whitespace/context
                     context_lcl = 3
                     ign_whitespace_lcl = False
 
                     for commit in c.commit_ranges:
                         commit2 = commit
                         commit1 = commit.first_parent
 
                         range_diff_cache_file_path = diff_cache_exist(
                             cache_path, 'diff', commit.raw_id,
                             ign_whitespace_lcl, context_lcl, c.fulldiff)
 
                         cached_diff = None
                         if caching_enabled:
                             cached_diff = load_cached_diff(range_diff_cache_file_path)
 
                         has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                         if not force_recache and has_proper_diff_cache:
                             diffset = cached_diff['diff']
                         else:
                             diffset = self._get_range_diffset(
                                 commits_source_repo, source_repo,
                                 commit1, commit2, diff_limit, file_limit,
                                 c.fulldiff, ign_whitespace_lcl, context_lcl
                             )
 
                         # save cached diff
                         if caching_enabled:
                             cache_diff(range_diff_cache_file_path, diffset, None)
 
                         c.changes[commit.raw_id] = diffset
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
 
             def mark(cs, fw):
-                return list(h.itertools.izip_longest([], cs, fillvalue=fw))
+                return list(h.itertools.zip_longest([], cs, fillvalue=fw))
 
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
             is_reviewer = PullRequestModel().is_user_reviewer(
                 pull_request, self._rhodecode_user)
             if is_reviewer:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm,
             maybe_unreachable=False):
 
         commit_cache = collections.OrderedDict()
         missing_requirements = False
 
         try:
             pre_load = ["author", "date", "message", "branch", "parents"]
 
             pull_request_commits = pull_request_at_ver.revisions
             log.debug('Loading %s commits from %s',
                       len(pull_request_commits), commits_source_repo)
 
             for rev in pull_request_commits:
                 comm = commits_source_repo.get_commit(commit_id=rev, pre_load=pre_load,
                                                       maybe_unreachable=maybe_unreachable)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         except CommitDoesNotExistError:
             log.warning('Failed to get commit from `{}` repo'.format(
                 commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning('Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
 
         pr_ancestor_id = pull_request_at_ver.common_ancestor_id
 
         try:
             ancestor_commit = source_scm.get_commit(pr_ancestor_id)
         except Exception:
             ancestor_commit = None
 
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent and c.has_origin_repo_read_perm:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
             'source_refs_json': json.dumps(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
             'target_refs_json': json.dumps(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_repo_targets(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         # get the parents
         parent_target_repos = []
         if self.db_repo.parent:
             parents_query = Repository.query() \
                 .order_by(func.length(Repository.repo_name)) \
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
                 ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()
 
             for parent in parents:
                 parent_vcs_obj = parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     parent_target_repos.append(parent)
 
         # get other forks, and repo itself
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_id == self.db_repo.repo_id,  # repo itself
                     Repository.fork_id == self.db_repo.repo_id)  # forks of this repo
             )  \
             .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(Repository.repo_name.ilike(ilike_expression))
 
         limit = max(20 - len(parent_target_repos), 5)  # not less then 5
         target_repos = query.limit(limit).all()
 
         all_target_repos = target_repos + parent_target_repos
 
         repos = []
         # This checks permissions to the repositories
         for obj in ScmModel().get_repos(all_target_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @classmethod
     def get_comment_ids(cls, post_data):
         return filter(lambda e: e > 0, map(safe_int, aslist(post_data.get('comments'), ',')))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_comments(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         self.register_comments_vars(c, pull_request_latest, versions, include_drafts=False)
         all_comments = c.inline_comments_flat + c.comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_todos(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request, include_drafts=False)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request, include_drafts=False)
 
         all_comments = c.unresolved_comments + c.resolved_comments
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(c.unresolved_comments),
                        todo_comments=True, existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_drafts(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.draft_comments = CommentsModel() \
             .get_pull_request_drafts(self._rhodecode_db_user.user_id, pull_request)
 
         all_comments = c.draft_comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids, draft_comments=True)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
         common_ancestor_id = _form['common_ancestor']
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         if not (source_db_repo or target_db_repo):
             h.flash(_('source_repo or target repo not found'), category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_ref_obj = unicode_to_reference(source_ref)
         target_ref_obj = unicode_to_reference(target_ref)
 
         source_commit = source_scm.get_commit(source_ref_obj.commit_id)
         target_commit = target_scm.get_commit(target_ref_obj.commit_id)
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref = ':'.join((target_ref_obj.type, target_ref_obj.name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user,
             source_db_repo,
             source_ref_obj,
             target_db_repo,
             target_ref_obj,
             include_diff_info=False)
 
         reviewers = validate_default_reviewers(_form['review_members'], reviewer_rules)
         observers = validate_observers(_form['observer_members'], reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref_obj.name
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
         description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 common_ancestor_id=common_ancestor_id,
                 reviewers=reviewers,
                 observers=observers,
                 title=pullrequest_title,
                 description=description,
                 description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         c = self.load_default_context()
         redirect_url = None
         # we do this check as first, because we want to know ASAP in the flow that
         # pr is updating currently
         is_state_changing = pull_request.is_state_changing()
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return {'response': True,
                     'redirect_url': redirect_url}
 
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
 
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
             force_refresh = str2bool(self.request.POST.get('force_refresh', 'false'))
             do_update_commits = str2bool(self.request.POST.get('update_commits', 'false'))
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_REVIEWER)
             elif 'observer_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['observer_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_OBSERVER)
             elif do_update_commits:
                 if is_state_changing:
                     log.debug('commits update: forbidden because pull request is in state %s',
                               pull_request.pull_request_state)
                     msg = _(u'Cannot update pull requests commits in state other than `{}`. '
                             u'Current state is: `{}`').format(
                         PullRequest.STATE_CREATED, pull_request.pull_request_state)
                     h.flash(msg, category='error')
                     return {'response': True,
                             'redirect_url': redirect_url}
 
                 self._update_commits(c, pull_request)
                 if force_refresh:
                     redirect_url = h.route_path(
                         'pullrequest_show', repo_name=self.db_repo_name,
                         pull_request_id=pull_request.pull_request_id,
                         _query={"force_refresh": 1})
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 log.error('Unhandled update data.')
                 raise HTTPBadRequest()
 
             return {'response': True,
                     'redirect_url': redirect_url}
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         """
         Edit title and description
         """
         _ = self.request.translate
 
         try:
             PullRequestModel().edit(
                 pull_request,
                 self.request.POST.get('title'),
                 self.request.POST.get('description'),
                 self.request.POST.get('description_renderer'),
                 self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
     def _update_commits(self, c, pull_request):
         _ = self.request.translate
         log.debug('pull-request: running update commits actions')
 
         @retry(exception=Exception, n_tries=3, delay=2)
         def commits_update():
             return PullRequestModel().update_commits(
                 pull_request, self._rhodecode_db_user)
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             resp = commits_update()  # retry x3
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(u'Pull request updated to "{source_commit_id}" with '
                     u'{count_added} added, {count_removed} removed commits. '
                     u'Source of changes: {change_source}.')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     def _update_reviewers(self, c, pull_request, review_members, reviewer_rules, role):
         _ = self.request.translate
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         if role == PullRequestReviewers.ROLE_REVIEWER:
             try:
                 reviewers = validate_default_reviewers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Reviewers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             old_calculated_status = pull_request.calculated_review_status()
             PullRequestModel().update_reviewers(
                 pull_request, reviewers, self._rhodecode_db_user)
 
             Session().commit()
 
             msg = _('Pull request reviewers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
             # trigger status changed if change in reviewers changes the status
             calculated_status = pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
 
         elif role == PullRequestReviewers.ROLE_OBSERVER:
             try:
                 observers = validate_observers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Observers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             PullRequestModel().update_observers(
                 pull_request, observers, self._rhodecode_db_user)
 
             Session().commit()
             msg = _('Pull request observers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         if pull_request.is_state_changing():
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot merge pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_show',
                              repo_name=pull_request.target_repo.repo_name,
                              pull_request_id=pull_request.pull_request_id))
 
         self.load_default_context()
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             check = MergeCheck.validate(
                 pull_request, auth_user=self._rhodecode_user,
                 translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             with pull_request.set_state(PullRequest.STATE_UPDATING):
                 self._merge_pull_request(
                     pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s", merge_resp)
             msg = merge_resp.merge_status_message
             h.flash(msg, category='error')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     def _pull_request_comments_create(self, pull_request, comments):
         _ = self.request.translate
         data = {}
         if not comments:
             return
         pull_request_id = pull_request.pull_request_id
 
         all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
 
         for entry in comments:
             c = self.load_default_context()
             comment_type = entry['comment_type']
             text = entry['text']
             status = entry['status']
             is_draft = str2bool(entry['is_draft'])
             resolves_comment_id = entry['resolves_comment_id']
             close_pull_request = entry['close_pull_request']
             f_path = entry['f_path']
             line_no = entry['line']
             target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
 
             # the logic here should work like following, if we submit close
             # pr comment, use `close_pull_request_with_comment` function
             # else handle regular comment logic
 
             if close_pull_request:
                 # only owner or admin or person with write permissions
                 allowed_to_close = PullRequestModel().check_user_update(
                     pull_request, self._rhodecode_user)
                 if not allowed_to_close:
                     log.debug('comment: forbidden because not allowed to close '
                               'pull request %s', pull_request_id)
                     raise HTTPForbidden()
 
                 # This also triggers `review_status_change`
                 comment, status = PullRequestModel().close_pull_request_with_comment(
                     pull_request, self._rhodecode_user, self.db_repo, message=text,
                     auth_user=self._rhodecode_user)
                 Session().flush()
                 is_inline = comment.is_inline
 
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment',
                     data={'comment': comment})
 
             else:
                 # regular comment case, could be inline, or one with status.
                 # for that one we check also permissions
                 # Additionally ENSURE if somehow draft is sent we're then unable to change status
                 allowed_to_change_status = PullRequestModel().check_user_change_status(
                     pull_request, self._rhodecode_user) and not is_draft
 
                 if status and allowed_to_change_status:
                     message = (_('Status change %(transition_icon)s %(status)s')
                                % {'transition_icon': '>',
                                   'status': ChangesetStatus.get_status_lbl(status)})
                     text = text or message
 
                 comment = CommentsModel().create(
                     text=text,
                     repo=self.db_repo.repo_id,
                     user=self._rhodecode_user.user_id,
                     pull_request=pull_request,
                     f_path=f_path,
                     line_no=line_no,
                     status_change=(ChangesetStatus.get_status_lbl(status)
                                    if status and allowed_to_change_status else None),
                     status_change_type=(status
                                         if status and allowed_to_change_status else None),
                     comment_type=comment_type,
                     is_draft=is_draft,
                     resolves_comment_id=resolves_comment_id,
                     auth_user=self._rhodecode_user,
                     send_email=not is_draft,  # skip notification for draft comments
                 )
                 is_inline = comment.is_inline
 
                 if allowed_to_change_status:
                     # calculate old status before we change it
                     old_calculated_status = pull_request.calculated_review_status()
 
                     # get status if set !
                     if status:
                         ChangesetStatusModel().set_status(
                             self.db_repo.repo_id,
                             status,
                             self._rhodecode_user.user_id,
                             comment,
                             pull_request=pull_request
                         )
 
                     Session().flush()
                     # this is somehow required to get access to some relationship
                     # loaded on comment
                     Session().refresh(comment)
 
                     # skip notifications for drafts
                     if not is_draft:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'comment',
                             data={'comment': comment})
 
                     # we now calculate the status of pull request, and based on that
                     # calculation we set the commits status
                     calculated_status = pull_request.calculated_review_status()
                     if old_calculated_status != calculated_status:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'review_status_change',
                             data={'status': calculated_status})
 
             comment_id = comment.comment_id
             data[comment_id] = {
                 'target_id': target_elem_id
             }
             Session().flush()
 
             c.co = comment
             c.at_version_num = None
             c.is_new = True
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data[comment_id].update(comment.get_dict())
             data[comment_id].update({'rendered_text': rendered_comment})
 
         Session().commit()
 
         # skip channelstream for draft comments
         if not all_drafts:
             comment_broadcast_channel = channelstream.comment_channel(
                 self.db_repo_name, pull_request_obj=pull_request)
 
             comment_data = data
             posted_comment_type = 'inline' if is_inline else 'general'
             if len(data) == 1:
                 msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
             else:
                 msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
 
             channelstream.comment_channelstream_push(
                 self.request, comment_broadcast_channel, self._rhodecode_user, msg,
                 comment_data=comment_data)
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(self.request.matchdict['pull_request_id'])
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug('comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         comment_data = {
             'comment_type': self.request.POST.get('comment_type'),
             'text': self.request.POST.get('text'),
             'status': self.request.POST.get('changeset_status', None),
             'is_draft': self.request.POST.get('draft'),
             'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
             'close_pull_request': self.request.POST.get('close_pull_request'),
             'f_path': self.request.POST.get('f_path'),
             'line': self.request.POST.get('line'),
         }
         data = self._pull_request_comments_create(pull_request, [comment_data])
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if comment.draft and not comment_owner:
             # We never allow to delete draft comments for other than owners
             raise HTTPNotFound()
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_edit(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id']
         )
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id']
         )
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(PR): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(PR): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             Session().commit()
             if not comment.draft:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment_edit',
                     data={'comment': comment})
 
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()

rhodecode/apps/repository/views/repo_settings_issue_trackers.py

@@ -1,127 +1,125 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 import formencode
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.lib import audit_logger
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.model.forms import IssueTrackerPatternsForm
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSettingsIssueTrackersView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     def repo_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
         c.data = 'data'
 
         c.settings_model = self.db_repo_patterns
         c.global_patterns = c.settings_model.get_global_settings()
         c.repo_patterns = c.settings_model.get_repo_settings()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_test(self):
         return h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
             self.db_repo_name)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_delete(self):
         _ = self.request.translate
         uid = self.request.POST.get('uid')
         repo_settings = self.db_repo_patterns
         try:
             repo_settings.delete_entries(uid)
         except Exception:
             h.flash(_('Error occurred during deleting issue tracker entry'),
                     category='error')
             raise HTTPNotFound()
 
         SettingsModel().invalidate_settings_cache()
         h.flash(_('Removed issue tracker entry.'), category='success')
 
         return {'deleted': uid}
 
     def _update_patterns(self, form, repo_settings):
         for uid in form['delete_patterns']:
             repo_settings.delete_entries(uid)
 
         for pattern_data in form['patterns']:
             for setting_key, pattern, type_ in pattern_data:
                 sett = repo_settings.create_or_update_setting(
                     setting_key, pattern.strip(), type_)
                 Session().add(sett)
 
             Session().commit()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_update(self):
         _ = self.request.translate
         # Save inheritance
         repo_settings = self.db_repo_patterns
         inherited = (
             self.request.POST.get('inherit_global_issuetracker') == "inherited")
         repo_settings.inherit_global_settings = inherited
         Session().commit()
 
         try:
             form = IssueTrackerPatternsForm(self.request.translate)().to_python(self.request.POST)
         except formencode.Invalid as errors:
             log.exception('Failed to add new pattern')
             error = errors
             h.flash(_('Invalid issue tracker pattern: {}'.format(error)),
                     category='error')
             raise HTTPFound(
                 h.route_path('edit_repo_issuetracker',
                              repo_name=self.db_repo_name))
 
         if form:
             self._update_patterns(form, repo_settings)
 
         h.flash(_('Updated issue tracker entries'), category='success')
         raise HTTPFound(
             h.route_path('edit_repo_issuetracker', repo_name=self.db_repo_name))
 

rhodecode/apps/repository/views/repo_summary.py

@@ -1,290 +1,290 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import string
 import time
 
 import rhodecode
 
 
 
 from rhodecode.lib.view_utils import get_format_ref_id
 from rhodecode.apps._base import RepoAppView
 from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
 from rhodecode.lib import helpers as h, rc_cache
 from rhodecode.lib.utils2 import safe_str, safe_int
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     CommitError, EmptyRepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import Statistics, CacheKey, User
 from rhodecode.model.meta import Session
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSummaryView(RepoAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = None
         if not c.repository_requirements_missing:
             c.rhodecode_repo = self.rhodecode_vcs_repo
         return c
 
     def _load_commits_context(self, c):
         p = safe_int(self.request.GET.get('page'), 1)
         size = safe_int(self.request.GET.get('size'), 10)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num,
                 'size': size
             }
             return h.route_path(
                 'repo_summary_commits',
                 repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)
 
         pre_load = self.get_commit_preload_attrs()
 
         try:
             collection = self.rhodecode_vcs_repo.get_commits(
                 pre_load=pre_load, translate_tags=False)
         except EmptyRepositoryError:
             collection = self.rhodecode_vcs_repo
 
         c.repo_commits = h.RepoPage(
             collection, page=p, items_per_page=size, url_maker=url_generator)
         page_ids = [x.raw_id for x in c.repo_commits]
         c.comments = self.db_repo.get_comments(page_ids)
         c.statuses = self.db_repo.statuses(page_ids)
 
     def _prepare_and_set_clone_url(self, c):
         username = ''
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = safe_str(self._rhodecode_user.username)
 
         _def_clone_uri = c.clone_uri_tmpl
         _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri_id)
         c.clone_repo_url_ssh = self.db_repo.clone_url(
             uri_tmpl=_def_clone_uri_ssh, ssh=True)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def summary_commits(self):
         c = self.load_default_context()
         self._prepare_and_set_clone_url(c)
         self._load_commits_context(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def summary(self):
         c = self.load_default_context()
 
         # Prepare the clone URL
         self._prepare_and_set_clone_url(c)
 
         # If enabled, get statistics data
         c.show_stats = bool(self.db_repo.enable_statistics)
 
         stats = Session().query(Statistics) \
             .filter(Statistics.repository == self.db_repo) \
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is self.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
 
             # Sort first by decreasing count and second by the file extension,
             # so we have a consistent output.
             lang_stats_items = sorted(lang_stats_d.items(),
                                       key=lambda k: (-k[1], k[0]))[:10]
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_items]
 
             c.trending_languages = json.dumps(lang_stats)
         else:
             c.no_data = True
             c.trending_languages = json.dumps({})
 
         scm_model = ScmModel()
         c.enable_downloads = self.db_repo.enable_downloads
         c.repository_followers = scm_model.get_followers(self.db_repo)
         c.repository_forks = scm_model.get_forks(self.db_repo)
 
         # first interaction with the VCS instance after here...
         if c.repository_requirements_missing:
             self.request.override_renderer = \
                 'rhodecode:templates/summary/missing_requirements.mako'
             return self._get_template_context(c)
 
         c.readme_data, c.readme_file = \
             self._get_readme_data(self.db_repo, c.visual.default_renderer)
 
         # loads the summary commits template context
         self._load_commits_context(c)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_stats(self):
         show_stats = bool(self.db_repo.enable_statistics)
         repo_id = self.db_repo.repo_id
 
         landing_commit = self.db_repo.get_landing_commit()
         if isinstance(landing_commit, EmptyCommit):
             return {'size': 0, 'code_stats': {}}
 
         cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = cache_seconds > 0
 
         log.debug(
             'Computing REPO STATS for repo_id %s commit_id `%s` '
             'with caching: %s[TTL: %ss]' % (
                 repo_id, landing_commit, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
         def compute_stats(repo_id, commit_id, _show_stats):
             code_stats = {}
             size = 0
             try:
                 commit = self.db_repo.get_commit(commit_id)
 
                 for node in commit.get_filenodes_generator():
                     size += node.size
                     if not _show_stats:
                         continue
-                    ext = string.lower(node.extension)
+                    ext = node.extension.lower()
                     ext_info = LANGUAGES_EXTENSIONS_MAP.get(ext)
                     if ext_info:
                         if ext in code_stats:
                             code_stats[ext]['count'] += 1
                         else:
                             code_stats[ext] = {"count": 1, "desc": ext_info}
             except (EmptyRepositoryError, CommitDoesNotExistError):
                 pass
             return {'size': h.format_byte_size_binary(size),
                     'code_stats': code_stats}
 
         stats = compute_stats(self.db_repo.repo_id, landing_commit.raw_id, show_stats)
         return stats
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_refs_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
         refs_to_create = [
             (_("Branch"), repo.branches, 'branch'),
             (_("Tag"), repo.tags, 'tag'),
             (_("Bookmark"), repo.bookmarks, 'book'),
         ]
         res = self._create_reference_data(repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_refs_changelog_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
 
         refs_to_create = [
             (_("Branches"), repo.branches, 'branch'),
             (_("Closed branches"), repo.branches_closed, 'branch_closed'),
             # TODO: enable when vcs can handle bookmarks filters
             # (_("Bookmarks"), repo.bookmarks, "book"),
         ]
         res = self._create_reference_data(
             repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     def _create_reference_data(self, repo, full_repo_name, refs_to_create):
         format_ref_id = get_format_ref_id(repo)
 
         result = []
         for title, refs, ref_type in refs_to_create:
             if refs:
                 result.append({
                     'text': title,
                     'children': self._create_reference_items(
                         repo, full_repo_name, refs, ref_type,
                         format_ref_id),
                 })
         return result
 
     def _create_reference_items(self, repo, full_repo_name, refs, ref_type, format_ref_id):
         result = []
         is_svn = h.is_svn(repo)
         for ref_name, raw_id in refs.items():
             files_url = self._create_files_url(
                 repo, full_repo_name, ref_name, raw_id, is_svn)
             result.append({
                 'text': ref_name,
                 'id': format_ref_id(ref_name, raw_id),
                 'raw_id': raw_id,
                 'type': ref_type,
                 'files_url': files_url,
                 'idx': 0,
             })
         return result
 
     def _create_files_url(self, repo, full_repo_name, ref_name, raw_id, is_svn):
         use_commit_id = '/' in ref_name or is_svn
         return h.route_path(
             'repo_files',
             repo_name=full_repo_name,
             f_path=ref_name if is_svn else '',
             commit_id=raw_id if use_commit_id else ref_name,
             _query=dict(at=ref_name))

rhodecode/authentication/plugins/auth_pam.py

@@ -1,172 +1,172 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication library for PAM
 """
 
 import colander
 import grp
 import logging
 import pam
 import pwd
 import re
 import socket
 
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, hybrid_property)
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwargs):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class PamAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class PamSettingsSchema(AuthnPluginSettingsSchemaBase):
     service = colander.SchemaNode(
         colander.String(),
         default='login',
         description=_('PAM service name to use for authentication.'),
         preparer=strip_whitespace,
         title=_('PAM service name'),
         widget='string')
     gecos = colander.SchemaNode(
         colander.String(),
-        default='(?P<last_name>.+),\s*(?P<first_name>\w+)',
+        default=r'(?P<last_name>.+),\s*(?P<first_name>\w+)',
         description=_('Regular expression for extracting user name/email etc. '
                       'from Unix userinfo.'),
         preparer=strip_whitespace,
         title=_('Gecos Regex'),
         widget='string')
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
     uid = 'pam'
     # PAM authentication can be slow. Repository operations involve a lot of
     # auth calls. Little caching helps speedup push/pull operations significantly
     AUTH_CACHE_TTL = 4
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), PamAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
             renderer='rhodecode:templates/admin/auth/plugin_settings.mako',
             request_method='GET',
             route_name='auth_home',
             context=PamAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
             renderer='rhodecode:templates/admin/auth/plugin_settings.mako',
             request_method='POST',
             route_name='auth_home',
             context=PamAuthnResource)
 
     def get_display_name(self, load_from_settings=False):
         return _('PAM')
 
     @classmethod
     def docs(cls):
         return "https://docs.rhodecode.com/RhodeCode-Enterprise/auth/auth-pam.html"
 
     @hybrid_property
     def name(self):
         return u"pam"
 
     def get_settings_schema(self):
         return PamSettingsSchema()
 
     def use_fake_password(self):
         return True
 
     def auth(self, userobj, username, password, settings, **kwargs):
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
         _pam = pam.pam()
         auth_result = _pam.authenticate(username, password, settings["service"])
 
         if not auth_result:
             log.error("PAM was unable to authenticate user: %s", username)
             return None
 
         log.debug('Got PAM response %s', auth_result)
 
         # old attrs fetched from RhodeCode database
         default_email = "%s@%s" % (username, socket.gethostname())
         admin = getattr(userobj, 'admin', False)
         active = getattr(userobj, 'active', True)
         email = getattr(userobj, 'email', '') or default_email
         username = getattr(userobj, 'username', username)
         firstname = getattr(userobj, 'firstname', '')
         lastname = getattr(userobj, 'lastname', '')
         extern_type = getattr(userobj, 'extern_type', '')
 
         user_attrs = {
             'username': username,
             'firstname': firstname,
             'lastname': lastname,
             'groups': [g.gr_name for g in grp.getgrall()
                        if username in g.gr_mem],
             'user_group_sync': True,
             'email': email,
             'admin': admin,
             'active': active,
             'active_from_extern': None,
             'extern_name': username,
             'extern_type': extern_type,
         }
 
         try:
             user_data = pwd.getpwnam(username)
             regex = settings["gecos"]
             match = re.search(regex, user_data.pw_gecos)
             if match:
                 user_attrs["firstname"] = match.group('first_name')
                 user_attrs["lastname"] = match.group('last_name')
         except Exception:
             log.warning("Cannot extract additional info for PAM user")
             pass
 
         log.debug("pamuser: %s", user_attrs)
         log.info('user `%s` authenticated correctly', user_attrs['username'],
                  extra={"action": "user_auth_ok", "auth_module": "auth_pam", "username": user_attrs["username"]})
         return user_attrs
 
 
 def includeme(config):
     plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
     plugin_factory(plugin_id).includeme(config)

rhodecode/config/environment.py

@@ -1,89 +1,90 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import logging
 import rhodecode
+import collections
 
 from rhodecode.config import utils
 
 from rhodecode.lib.utils import load_rcextensions
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib.vcs import connect_vcs
 
 log = logging.getLogger(__name__)
 
 
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
 
     # TODO(marcink): probably not required anymore
     # configure channelstream,
     settings_merged['channelstream_config'] = {
         'enabled': str2bool(settings_merged.get('channelstream.enabled', False)),
         'server': settings_merged.get('channelstream.server'),
         'secret': settings_merged.get('channelstream.secret')
     }
 
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
     if settings['is_test']:
         rhodecode.is_test = True
         rhodecode.disable_error_handler = True
         from rhodecode import authentication
         authentication.plugin_default_auth_ttl = 0
 
         utils.initialize_test_environment(settings_merged)
 
     # Initialize the database connection.
     utils.initialize_database(settings_merged)
 
     load_rcextensions(root_path=settings_merged['here'])
 
     # Limit backends to `vcs.backends` from configuration, and preserve the order
     for alias in rhodecode.BACKENDS.keys():
         if alias not in settings['vcs.backends']:
             del rhodecode.BACKENDS[alias]
 
     _sorted_backend = sorted(rhodecode.BACKENDS.items(),
                              key=lambda item: settings['vcs.backends'].index(item[0]))
-    rhodecode.BACKENDS = rhodecode.OrderedDict(_sorted_backend)
+    rhodecode.BACKENDS = collections.OrderedDict(_sorted_backend)
 
     log.info('Enabled VCS backends: %s', rhodecode.BACKENDS.keys())
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = settings['vcs.server']
     vcs_server_enabled = settings['vcs.server.enable']
 
     utils.configure_vcs(settings)
 
     # Store the settings to make them available to other modules.
 
     rhodecode.PYRAMID_SETTINGS = settings_merged
     rhodecode.CONFIG = settings_merged
     rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
     else:
         log.warning('vcs-server not enabled, vcs connection unavailable')

rhodecode/lib/_vendor/authomatic/adapters.py

@@ -1,282 +1,280 b''
 # -*- coding: utf-8 -*-
 """
 Adapters
 --------
 
 .. contents::
    :backlinks: none
 
 The :func:`authomatic.login` function needs access to functionality like
 getting the **URL** of the handler where it is being called, getting the
 **request params** and **cookies** and **writing the body**, **headers**
 and **status** to the response.
 
 Since implementation of these features varies across Python web frameworks,
 the Authomatic library uses **adapters** to unify these differences into a
 single interface.
 
 Available Adapters
 ^^^^^^^^^^^^^^^^^^
 
 If you are missing an adapter for the framework of your choice, please
 open an `enhancement issue <https://github.com/authomatic/authomatic/issues>`_
 or consider a contribution to this module by
 :ref:`implementing <implement_adapters>` one by yourself.
 Its very easy and shouldn't take you more than a few minutes.
 
 .. autoclass:: DjangoAdapter
     :members:
 
 .. autoclass:: Webapp2Adapter
     :members:
 
 .. autoclass:: WebObAdapter
     :members:
 
 .. autoclass:: WerkzeugAdapter
     :members:
 
 .. _implement_adapters:
 
 Implementing an Adapter
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 Implementing an adapter for a Python web framework is pretty easy.
 
 Do it by subclassing the :class:`.BaseAdapter` abstract class.
 There are only **six** members that you need to implement.
 
 Moreover if your framework is based on the |webob|_ or |werkzeug|_ package
 you can subclass the :class:`.WebObAdapter` or :class:`.WerkzeugAdapter`
 respectively.
 
 .. autoclass:: BaseAdapter
     :members:
 
 """
 
 import abc
 from authomatic.core import Response
 
 
-class BaseAdapter(object):
+class BaseAdapter(object, metaclass=abc.ABCMeta):
     """
     Base class for platform adapters.
 
     Defines common interface for WSGI framework specific functionality.
 
     """
 
-    __metaclass__ = abc.ABCMeta
-
     @abc.abstractproperty
     def params(self):
         """
         Must return a :class:`dict` of all request parameters of any HTTP
         method.
 
         :returns:
             :class:`dict`
 
         """
 
     @abc.abstractproperty
     def url(self):
         """
         Must return the url of the actual request including path but without
         query and fragment.
 
         :returns:
             :class:`str`
 
         """
 
     @abc.abstractproperty
     def cookies(self):
         """
         Must return cookies as a :class:`dict`.
 
         :returns:
             :class:`dict`
 
         """
 
     @abc.abstractmethod
     def write(self, value):
         """
         Must write specified value to response.
 
         :param str value:
             String to be written to response.
 
         """
 
     @abc.abstractmethod
     def set_header(self, key, value):
         """
         Must set response headers to ``Key: value``.
 
         :param str key:
             Header name.
 
         :param str value:
             Header value.
 
         """
 
     @abc.abstractmethod
     def set_status(self, status):
         """
         Must set the response status e.g. ``'302 Found'``.
 
         :param str status:
             The HTTP response status.
 
         """
 
 
 class DjangoAdapter(BaseAdapter):
     """
     Adapter for the |django|_ framework.
     """
 
     def __init__(self, request, response):
         """
         :param request:
             An instance of the :class:`django.http.HttpRequest` class.
 
         :param response:
             An instance of the :class:`django.http.HttpResponse` class.
         """
         self.request = request
         self.response = response
 
     @property
     def params(self):
         params = {}
         params.update(self.request.GET.dict())
         params.update(self.request.POST.dict())
         return params
 
     @property
     def url(self):
         return self.request.build_absolute_uri(self.request.path)
 
     @property
     def cookies(self):
         return dict(self.request.COOKIES)
 
     def write(self, value):
         self.response.write(value)
 
     def set_header(self, key, value):
         self.response[key] = value
 
     def set_status(self, status):
         status_code, reason = status.split(' ', 1)
         self.response.status_code = int(status_code)
 
 
 class WebObAdapter(BaseAdapter):
     """
     Adapter for the |webob|_ package.
     """
 
     def __init__(self, request, response):
         """
         :param request:
             A |webob|_ :class:`Request` instance.
 
         :param response:
             A |webob|_ :class:`Response` instance.
         """
         self.request = request
         self.response = response
 
     # =========================================================================
     # Request
     # =========================================================================
 
     @property
     def url(self):
         return self.request.path_url
 
     @property
     def params(self):
         return dict(self.request.params)
 
     @property
     def cookies(self):
         return dict(self.request.cookies)
 
     # =========================================================================
     # Response
     # =========================================================================
 
     def write(self, value):
         self.response.write(value)
 
     def set_header(self, key, value):
         self.response.headers[key] = str(value)
 
     def set_status(self, status):
         self.response.status = status
 
 
 class Webapp2Adapter(WebObAdapter):
     """
     Adapter for the |webapp2|_ framework.
 
     Inherits from the :class:`.WebObAdapter`.
 
     """
 
     def __init__(self, handler):
         """
         :param handler:
             A :class:`webapp2.RequestHandler` instance.
         """
         self.request = handler.request
         self.response = handler.response
 
 
 class WerkzeugAdapter(BaseAdapter):
     """
     Adapter for |flask|_ and other |werkzeug|_ based frameworks.
 
     Thanks to `Mark Steve Samson <http://marksteve.com>`_.
 
     """
 
     @property
     def params(self):
         return self.request.args
 
     @property
     def url(self):
         return self.request.base_url
 
     @property
     def cookies(self):
         return self.request.cookies
 
     def __init__(self, request, response):
         """
         :param request:
             Instance of the :class:`werkzeug.wrappers.Request` class.
 
         :param response:
             Instance of the :class:`werkzeug.wrappers.Response` class.
         """
 
         self.request = request
         self.response = response
 
     def write(self, value):
         self.response.data = self.response.data + value
 
     def set_header(self, key, value):
         self.response.headers[key] = value
 
     def set_status(self, status):
         self.response.status = status

rhodecode/lib/audit_logger.py

@@ -1,305 +1,305 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 
 from rhodecode.lib.jsonalchemy import JsonRaw
 from rhodecode.model import meta
 from rhodecode.model.db import User, UserLog, Repository
 
 
 log = logging.getLogger(__name__)
 
 # action as key, and expected action_data as value
 ACTIONS_V1 = {
     'user.login.success': {'user_agent': ''},
     'user.login.failure': {'user_agent': ''},
     'user.logout': {'user_agent': ''},
     'user.register': {},
     'user.password.reset_request': {},
     'user.push': {'user_agent': '', 'commit_ids': []},
     'user.pull': {'user_agent': ''},
 
     'user.create': {'data': {}},
     'user.delete': {'old_data': {}},
     'user.edit': {'old_data': {}},
     'user.edit.permissions': {},
     'user.edit.ip.add': {'ip': {}, 'user': {}},
     'user.edit.ip.delete': {'ip': {}, 'user': {}},
     'user.edit.token.add': {'token': {}, 'user': {}},
     'user.edit.token.delete': {'token': {}, 'user': {}},
     'user.edit.email.add': {'email': ''},
     'user.edit.email.delete': {'email': ''},
     'user.edit.ssh_key.add': {'token': {}, 'user': {}},
     'user.edit.ssh_key.delete': {'token': {}, 'user': {}},
     'user.edit.password_reset.enabled': {},
     'user.edit.password_reset.disabled': {},
 
     'user_group.create': {'data': {}},
     'user_group.delete': {'old_data': {}},
     'user_group.edit': {'old_data': {}},
     'user_group.edit.permissions': {},
     'user_group.edit.member.add': {'user': {}},
     'user_group.edit.member.delete': {'user': {}},
 
     'repo.create': {'data': {}},
     'repo.fork': {'data': {}},
     'repo.edit': {'old_data': {}},
     'repo.edit.permissions': {},
     'repo.edit.permissions.branch': {},
     'repo.archive': {'old_data': {}},
     'repo.delete': {'old_data': {}},
 
     'repo.archive.download': {'user_agent': '', 'archive_name': '',
                               'archive_spec': '', 'archive_cached': ''},
 
     'repo.permissions.branch_rule.create': {},
     'repo.permissions.branch_rule.edit': {},
     'repo.permissions.branch_rule.delete': {},
 
     'repo.pull_request.create': '',
     'repo.pull_request.edit': '',
     'repo.pull_request.delete': '',
     'repo.pull_request.close': '',
     'repo.pull_request.merge': '',
     'repo.pull_request.vote': '',
     'repo.pull_request.comment.create': '',
     'repo.pull_request.comment.edit': '',
     'repo.pull_request.comment.delete': '',
 
     'repo.pull_request.reviewer.add': '',
     'repo.pull_request.reviewer.delete': '',
 
     'repo.pull_request.observer.add': '',
     'repo.pull_request.observer.delete': '',
 
     'repo.commit.strip': {'commit_id': ''},
     'repo.commit.comment.create': {'data': {}},
     'repo.commit.comment.delete': {'data': {}},
     'repo.commit.comment.edit': {'data': {}},
     'repo.commit.vote': '',
 
     'repo.artifact.add': '',
     'repo.artifact.delete': '',
 
     'repo_group.create': {'data': {}},
     'repo_group.edit': {'old_data': {}},
     'repo_group.edit.permissions': {},
     'repo_group.delete': {'old_data': {}},
 }
 
 ACTIONS = ACTIONS_V1
 
 SOURCE_WEB = 'source_web'
 SOURCE_API = 'source_api'
 
 
 class UserWrap(object):
     """
     Fake object used to imitate AuthUser
     """
 
     def __init__(self, user_id=None, username=None, ip_addr=None):
         self.user_id = user_id
         self.username = username
         self.ip_addr = ip_addr
 
 
 class RepoWrap(object):
     """
     Fake object used to imitate RepoObject that audit logger requires
     """
 
     def __init__(self, repo_id=None, repo_name=None):
         self.repo_id = repo_id
         self.repo_name = repo_name
 
 
 def _store_log(action_name, action_data, user_id, username, user_data,
                ip_address, repository_id, repository_name):
     user_log = UserLog()
     user_log.version = UserLog.VERSION_2
 
     user_log.action = action_name
-    user_log.action_data = action_data or JsonRaw(u'{}')
+    user_log.action_data = action_data or JsonRaw('{}')
 
     user_log.user_ip = ip_address
 
     user_log.user_id = user_id
     user_log.username = username
-    user_log.user_data = user_data or JsonRaw(u'{}')
+    user_log.user_data = user_data or JsonRaw('{}')
 
     user_log.repository_id = repository_id
     user_log.repository_name = repository_name
 
     user_log.action_date = datetime.datetime.now()
 
     return user_log
 
 
 def store_web(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_WEB
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store_api(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_API
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store(action, user, action_data=None, user_data=None, ip_addr=None,
           repo=None, sa_session=None, commit=False):
     """
     Audit logger for various actions made by users, typically this
     results in a call such::
 
         from rhodecode.lib import audit_logger
 
         audit_logger.store(
             'repo.edit', user=self._rhodecode_user)
         audit_logger.store(
             'repo.delete', action_data={'data': repo_data},
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'))
 
         # repo action
         audit_logger.store(
             'repo.delete',
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'),
             repo=audit_logger.RepoWrap(repo_name='some-repo'))
 
         # repo action, when we know and have the repository object already
         audit_logger.store(
             'repo.delete', action_data={'source': audit_logger.SOURCE_WEB, },
             user=self._rhodecode_user,
             repo=repo_object)
 
         # alternative wrapper to the above
         audit_logger.store_web(
             'repo.delete', action_data={},
             user=self._rhodecode_user,
             repo=repo_object)
 
         # without an user ?
         audit_logger.store(
             'user.login.failure',
             user=audit_logger.UserWrap(
                     username=self.request.params.get('username'),
                     ip_addr=self.request.remote_addr))
 
     """
     from rhodecode.lib.utils2 import safe_unicode
     from rhodecode.lib.auth import AuthUser
 
     action_spec = ACTIONS.get(action, None)
     if action_spec is None:
         raise ValueError('Action `{}` is not supported'.format(action))
 
     if not sa_session:
         sa_session = meta.Session()
 
     try:
         username = getattr(user, 'username', None)
         if not username:
             pass
 
         user_id = getattr(user, 'user_id', None)
         if not user_id:
             # maybe we have username ? Try to figure user_id from username
             if username:
                 user_id = getattr(
                     User.get_by_username(username), 'user_id', None)
 
         ip_addr = ip_addr or getattr(user, 'ip_addr', None)
         if not ip_addr:
             pass
 
         if not user_data:
             # try to get this from the auth user
             if isinstance(user, AuthUser):
                 user_data = {
                     'username': user.username,
                     'email': user.email,
                 }
 
         repository_name = getattr(repo, 'repo_name', None)
         repository_id = getattr(repo, 'repo_id', None)
         if not repository_id:
             # maybe we have repo_name ? Try to figure repo_id from repo_name
             if repository_name:
                 repository_id = getattr(
                     Repository.get_by_repo_name(repository_name), 'repo_id', None)
 
         action_name = safe_unicode(action)
         ip_address = safe_unicode(ip_addr)
 
         with sa_session.no_autoflush:
 
             user_log = _store_log(
                 action_name=action_name,
                 action_data=action_data or {},
                 user_id=user_id,
                 username=username,
                 user_data=user_data or {},
                 ip_address=ip_address,
                 repository_id=repository_id,
                 repository_name=repository_name
             )
 
             sa_session.add(user_log)
             if commit:
                 sa_session.commit()
             entry_id = user_log.entry_id or ''
 
             update_user_last_activity(sa_session, user_id)
 
             if commit:
                 sa_session.commit()
 
         log.info('AUDIT[%s]: Logging action: `%s` by user:id:%s[%s] ip:%s',
                  entry_id, action_name, user_id, username, ip_address,
                  extra={"entry_id": entry_id, "action": action_name,
                         "user_id": user_id, "ip": ip_address})
 
     except Exception:
         log.exception('AUDIT: failed to store audit log')
 
 
 def update_user_last_activity(sa_session, user_id):
     _last_activity = datetime.datetime.now()
     try:
         sa_session.query(User).filter(User.user_id == user_id).update(
             {"last_activity": _last_activity})
         log.debug(
             'updated user `%s` last activity to:%s', user_id, _last_activity)
     except Exception:
         log.exception("Failed last activity update for user_id: %s", user_id)
         sa_session.rollback()
 

rhodecode/lib/channelstream.py

@@ -1,371 +1,371 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import itsdangerous
 import logging
 import requests
 import datetime
 
 from dogpile.util.readwrite_lock import ReadWriteMutex
 from pyramid.threadlocal import get_current_registry
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.ext_json import json
 from rhodecode.model.db import User
 
 log = logging.getLogger(__name__)
 
 LOCK = ReadWriteMutex()
 
 USER_STATE_PUBLIC_KEYS = [
     'id', 'username', 'first_name', 'last_name',
     'icon_link', 'display_name', 'display_link']
 
 
 class ChannelstreamException(Exception):
     pass
 
 
 class ChannelstreamConnectionException(ChannelstreamException):
     pass
 
 
 class ChannelstreamPermissionException(ChannelstreamException):
     pass
 
 
 def get_channelstream_server_url(config, endpoint):
     return 'http://{}{}'.format(config['server'], endpoint)
 
 
 def channelstream_request(config, payload, endpoint, raise_exc=True):
     signer = itsdangerous.TimestampSigner(config['secret'])
     sig_for_server = signer.sign(endpoint)
     secret_headers = {'x-channelstream-secret': sig_for_server,
                       'x-channelstream-endpoint': endpoint,
                       'Content-Type': 'application/json'}
     req_url = get_channelstream_server_url(config, endpoint)
 
     log.debug('Sending a channelstream request to endpoint: `%s`', req_url)
     response = None
     try:
         response = requests.post(req_url, data=json.dumps(payload),
                                  headers=secret_headers).json()
     except requests.ConnectionError:
         log.exception('ConnectionError occurred for endpoint %s', req_url)
         if raise_exc:
             raise ChannelstreamConnectionException(req_url)
     except Exception:
         log.exception('Exception related to Channelstream happened')
         if raise_exc:
             raise ChannelstreamConnectionException()
     log.debug('Got channelstream response: %s', response)
     return response
 
 
 def get_user_data(user_id):
     user = User.get(user_id)
     return {
         'id': user.user_id,
         'username': user.username,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'icon_link': h.gravatar_url(user.email, 60),
         'display_name': h.person(user, 'username_or_name_or_email'),
         'display_link': h.link_to_user(user),
         'notifications': user.user_data.get('notification_status', True)
     }
 
 
 def broadcast_validator(channel_name):
     """ checks if user can access the broadcast channel """
     if channel_name == 'broadcast':
         return True
 
 
 def repo_validator(channel_name):
     """ checks if user can access the broadcast channel """
     channel_prefix = '/repo$'
     if channel_name.startswith(channel_prefix):
         elements = channel_name[len(channel_prefix):].split('$')
         repo_name = elements[0]
         can_access = HasRepoPermissionAny(
             'repository.read',
             'repository.write',
             'repository.admin')(repo_name)
         log.debug(
             'permission check for %s channel resulted in %s',
             repo_name, can_access)
         if can_access:
             return True
     return False
 
 
 def check_channel_permissions(channels, plugin_validators, should_raise=True):
     valid_channels = []
 
     validators = [broadcast_validator, repo_validator]
     if plugin_validators:
         validators.extend(plugin_validators)
     for channel_name in channels:
         is_valid = False
         for validator in validators:
             if validator(channel_name):
                 is_valid = True
                 break
         if is_valid:
             valid_channels.append(channel_name)
         else:
             if should_raise:
                 raise ChannelstreamPermissionException()
     return valid_channels
 
 
 def get_channels_info(self, channels):
     payload = {'channels': channels}
     # gather persistence info
     return channelstream_request(self._config(), payload, '/info')
 
 
 def parse_channels_info(info_result, include_channel_info=None):
     """
     Returns data that contains only secure information that can be
     presented to clients
     """
     include_channel_info = include_channel_info or []
 
     user_state_dict = {}
     for userinfo in info_result['users']:
         user_state_dict[userinfo['user']] = {
             k: v for k, v in userinfo['state'].items()
             if k in USER_STATE_PUBLIC_KEYS
             }
 
     channels_info = {}
 
     for c_name, c_info in info_result['channels'].items():
         if c_name not in include_channel_info:
             continue
         connected_list = []
         for username in c_info['users']:
             connected_list.append({
                 'user': username,
                 'state': user_state_dict[username]
             })
         channels_info[c_name] = {'users': connected_list,
                                  'history': c_info['history']}
 
     return channels_info
 
 
 def log_filepath(history_location, channel_name):
     hasher = hashlib.sha256()
     hasher.update(channel_name.encode('utf8'))
     filename = '{}.log'.format(hasher.hexdigest())
     filepath = os.path.join(history_location, filename)
     return filepath
 
 
 def read_history(history_location, channel_name):
     filepath = log_filepath(history_location, channel_name)
     if not os.path.exists(filepath):
         return []
     history_lines_limit = -100
     history = []
     with open(filepath, 'rb') as f:
         for line in f.readlines()[history_lines_limit:]:
             try:
                 history.append(json.loads(line))
             except Exception:
                 log.exception('Failed to load history')
     return history
 
 
 def update_history_from_logs(config, channels, payload):
     history_location = config.get('history.location')
     for channel in channels:
         history = read_history(history_location, channel)
         payload['channels_info'][channel]['history'] = history
 
 
 def write_history(config, message):
     """ writes a message to a base64encoded filename """
     history_location = config.get('history.location')
     if not os.path.exists(history_location):
         return
     try:
         LOCK.acquire_write_lock()
         filepath = log_filepath(history_location, message['channel'])
         json_message = json.dumps(message)
         with open(filepath, 'ab') as f:
             f.write(json_message)
             f.write('\n')
     finally:
         LOCK.release_write_lock()
 
 
 def get_connection_validators(registry):
     validators = []
     for k, config in registry.rhodecode_plugins.items():
         validator = config.get('channelstream', {}).get('connect_validator')
         if validator:
             validators.append(validator)
     return validators
 
 
 def get_channelstream_config(registry=None):
     if not registry:
         registry = get_current_registry()
 
     rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
     channelstream_config = rhodecode_plugins.get('channelstream', {})
     return channelstream_config
 
 
 def post_message(channel, message, username, registry=None):
     channelstream_config = get_channelstream_config(registry)
     if not channelstream_config.get('enabled'):
         return
 
     message_obj = message
     if isinstance(message, str):
         message_obj = {
             'message': message,
             'level': 'success',
             'topic': '/notifications'
         }
 
     log.debug('Channelstream: sending notification to channel %s', channel)
     payload = {
         'type': 'message',
         'timestamp': datetime.datetime.utcnow(),
         'user': 'system',
         'exclude_users': [username],
         'channel': channel,
         'message': message_obj
     }
 
     try:
         return channelstream_request(
             channelstream_config, [payload], '/message',
             raise_exc=False)
     except ChannelstreamException:
         log.exception('Failed to send channelstream data')
         raise
 
 
 def _reload_link(label):
     return (
         '<a onclick="window.location.reload()">'
         '<strong>{}</strong>'
         '</a>'.format(label)
     )
 
 
 def pr_channel(pull_request):
     repo_name = pull_request.target_repo.repo_name
     pull_request_id = pull_request.pull_request_id
     channel = '/repo${}$/pr/{}'.format(repo_name, pull_request_id)
     log.debug('Getting pull-request channelstream broadcast channel: %s', channel)
     return channel
 
 
 def comment_channel(repo_name, commit_obj=None, pull_request_obj=None):
     channel = None
     if commit_obj:
-        channel = u'/repo${}$/commit/{}'.format(
+        channel = '/repo${}$/commit/{}'.format(
             repo_name, commit_obj.raw_id
         )
     elif pull_request_obj:
-        channel = u'/repo${}$/pr/{}'.format(
+        channel = '/repo${}$/pr/{}'.format(
             repo_name, pull_request_obj.pull_request_id
         )
     log.debug('Getting comment channelstream broadcast channel: %s', channel)
 
     return channel
 
 
 def pr_update_channelstream_push(request, pr_broadcast_channel, user, msg, **kwargs):
     """
     Channel push on pull request update
     """
     if not pr_broadcast_channel:
         return
 
     _ = request.translate
 
     message = '{} {}'.format(
         msg,
         _reload_link(_(' Reload page to load changes')))
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         pr_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
 
 def comment_channelstream_push(request, comment_broadcast_channel, user, msg, **kwargs):
     """
     Channelstream push on comment action, on commit, or pull-request
     """
     if not comment_broadcast_channel:
         return
 
     _ = request.translate
 
     comment_data = kwargs.pop('comment_data', {})
     user_data = kwargs.pop('user_data', {})
     comment_id = comment_data.keys()[0] if comment_data else ''
 
     message = '<strong>{}</strong> {} #{}'.format(
         user.username,
         msg,
         comment_id,
     )
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
     message_obj = {
         'message': None,
         'user': user.username,
         'comment_id': comment_id,
         'comment_data': comment_data,
         'user_data': user_data,
         'topic': '/comment'
     }
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)

rhodecode/lib/codeblocks.py

@@ -1,797 +1,797 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
 from itertools import groupby
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from pygments.lexers.special import TextLexer, Token
 from pygments.lexers import get_lexer_by_name
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict, safe_unicode
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer, DEL_FILENODE, BIN_FILENODE
 
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
     log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
               lexer, filenode, org_lexer)
     content = filenode.content
     tokens = tokenize_string(content, lexer)
     lines = split_token_stream(tokens, content)
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
 
     if isinstance(lexer, TextLexer):
         lexed = [(Token.Text, content)]
     else:
         lexed = lex(content, lexer)
 
     for token_type, token_text in lexed:
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens, content):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     token_buffer = []
     for token_class, token_text in tokens:
         parts = token_text.split('\n')
         for part in parts[:-1]:
             token_buffer.append((token_class, part))
             yield token_buffer
             token_buffer = []
 
         token_buffer.append((token_class, parts[-1]))
 
     if token_buffer:
         yield token_buffer
     elif content:
         # this is a special case, we have the content, but tokenization didn't produce
         # any results. THis can happen if know file extensions like .css have some bogus
         # unicode content without any newline characters
         yield [(pygment_token_class(Token.Text), content)]
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
-            result.append(u'<span class="%s">' % token_class)
+            result.append('<span class="%s">' % token_class)
         else:
-            result.append(u'<span>')
+            result.append('<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
-                result.append(u'<%s>' % op_tag)
+                result.append('<%s>' % op_tag)
 
             # NOTE(marcink): in some cases of mixed encodings, we might run into
             # troubles in the html_escape, in this case we say unicode force on token_text
             # that would ensure "correct" data even with the cost of rendered
             try:
                 escaped_text = html_escape(token_text)
             except TypeError:
                 escaped_text = html_escape(safe_unicode(token_text))
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
-                result.append(u'</%s>' % op_tag)
+                result.append('</%s>' % op_tag)
 
-        result.append(u'</span>')
+        result.append('</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
     for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
         ops = []
         for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
     similarity = difflib.SequenceMatcher(None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
     if similarity < 0.6: # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
     token_sequence_matcher = difflib.SequenceMatcher(None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
         # check the differences by token block types first to give a more
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, 'del', token_text))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, 'ins', token_text))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
                 for char in token_text:
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
                 for char in token_text:
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
                 token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
                 token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11  # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], 'del', c))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
                             nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 def diffset_node_getter(commit):
     def get_node(fname):
         try:
             return commit.get_node(fname)
         except NodeDoesNotExistError:
             return None
 
     return get_node
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL'  # highlights using original file, slow
     HL_FAST = 'FAST'  # highlights using just the line, fast but not correct
                       # in the case of multiline code
     HL_NONE = 'NONE'  # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_repo_name=None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  # files over this size will use fast highlighting
                  max_file_size_limit=150 * 1024,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {
             'before': {},
             'after': {}
         }
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.target_repo_name = target_repo_name or repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             target_repo_name=self.target_repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = StrictAttributeDict(dict(
                 source_ref=diffset.source_ref,
                 target_ref=diffset.target_ref,
                 repo_name=diffset.repo_name,
                 source_repo_name=diffset.source_repo_name,
                 target_repo_name=diffset.target_repo_name,
             ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r', patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             node_hl_mode = self.HL_NONE if patch['chunks'] == [] else None
             hl_mode = node_hl_mode or self.highlight_mode
 
             if hl_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif hl_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
         raw_id_uid = ''
         if self.source_nodes.get(source_filename):
             raw_id_uid = self.source_nodes[source_filename].commit.raw_id
 
         if not raw_id_uid and self.target_nodes.get(target_filename):
             # in case this is a new file we only have it in target
             raw_id_uid = self.target_nodes[target_filename].commit.raw_id
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': patch['is_limited_diff'],
             'hunks': [],
             'hunk_ops': None,
             'diffset': self,
             'raw_id': raw_id_uid,
         })
 
         file_chunks = patch['chunks'][1:]
         for i, hunk in enumerate(file_chunks, 1):
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             hunkbit.index = i
             filediff.hunks.append(hunkbit)
 
         # Simulate hunk on OPS type line which doesn't really contain any diff
         # this allows commenting on those
         if not file_chunks:
             actions = []
             for op_id, op_text in filediff.patch['stats']['ops'].items():
                 if op_id == DEL_FILENODE:
-                    actions.append(u'file was removed')
+                    actions.append('file was removed')
                 elif op_id == BIN_FILENODE:
-                    actions.append(u'binary diff hidden')
+                    actions.append('binary diff hidden')
                 else:
                     actions.append(safe_unicode(op_text))
-            action_line = u'NO CONTENT: ' + \
-                          u', '.join(actions) or u'UNDEFINED_ACTION'
+            action_line = 'NO CONTENT: ' + \
+                          ', '.join(actions) or 'UNDEFINED_ACTION'
 
             hunk_ops = {'source_length': 0, 'source_start': 0,
                         'lines': [
                             {'new_lineno': 0, 'old_lineno': 1,
                              'action': 'unmod-no-hl', 'line': action_line}
                         ],
-                        'section_header': u'', 'target_start': 1, 'target_length': 1}
+                        'section_header': '', 'target_start': 1, 'target_length': 1}
 
             hunkbit = self.parse_hunk(hunk_ops, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunk_ops = hunkbit
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
             if line['action'] in ['unmod', 'unmod-no-hl']:
                 no_hl = line['action'] == 'unmod-no-hl'
                 result.lines.extend(
                     self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
             elif line['action'] == 'new-no-nl':
                 after.append(line)
 
         all_actions = [x['action'] for x in after] + [x['action'] for x in before]
         no_hl = {x for x in all_actions} == {'unmod-no-hl'}
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
         # NOTE(marcink): we must keep list() call here so we can cache the result...
         result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file,
                     no_hl=False):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
 
         before_newline = AttributeDict()
         after_newline = AttributeDict()
         if before_lines and before_lines[-1]['action'] == 'old-no-nl':
             before_newline_line = before_lines.pop(-1)
             before_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', before_newline_line['line'])]]))
 
         if after_lines and after_lines[-1]['action'] == 'new-no-nl':
             after_newline_line = after_lines.pop(-1)
             after_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', after_newline_line['line'])]]))
 
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
                     before_tokens = [('nonl', before['line'])]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'], line_number=before['old_lineno'],
                         input_file=source_file, no_hl=no_hl, source='before')
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
 
                 original.get_comment_args = (
                     source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
                     after_tokens = [('nonl', after['line'])]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         input_file=target_file, no_hl=no_hl, source='after')
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
 
                 modified.get_comment_args = (target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
             if not before_lines and before_newline:
                 original.content += before_newline.content
                 before_newline = None
             if not after_lines and after_newline:
                 modified.content += after_newline.content
                 after_newline = None
 
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_line_tokens(self, line_text, line_number, input_file=None, no_hl=False, source=''):
         filenode = None
         filename = None
 
         if isinstance(input_file, str):
             filename = input_file
         elif isinstance(input_file, FileNode):
             filenode = input_file
             filename = input_file.unicode_path
 
         hl_mode = self.HL_NONE if no_hl else self.highlight_mode
         if hl_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
             file_size_allowed = input_file.size < self.max_file_size_limit
             if line_number and file_size_allowed:
                 return self.get_tokenized_filenode_line(input_file, line_number, lexer, source)
 
         if hl_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None, source=''):
 
         def tokenize(_filenode):
             self.highlighted_filenodes[source][filenode] = filenode_as_lines_tokens(filenode, lexer)
 
         if filenode not in self.highlighted_filenodes[source]:
             tokenize(filenode)
 
         try:
             return self.highlighted_filenodes[source][filenode][line_number - 1]
         except Exception:
             log.exception('diff rendering error')
-            return [('', u'L{}: rhodecode diff rendering error'.format(line_number))]
+            return [('', 'L{}: rhodecode diff rendering error'.format(line_number))]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'unmod-no-hl': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.get_comment_args)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/lib/db_manage.py

@@ -1,680 +1,680 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print(('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper())
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args=None):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def db_exists(self):
         if not self.sa:
             self.init_db()
         try:
             self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
             return True
         except Exception:
             return False
         finally:
             self.sa.rollback()
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('db tables bootstrap: Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def run_post_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         if version:
             DbMigrateVersion.set_version(version)
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database db_uri under version '
                    'control with version {}'.format(curr_version))
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to upgrade database from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         self.run_post_migration_tasks()
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == User.DEFAULT_USER)\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
-            username = input('Specify admin username:')
+            username = eval(input('Specify admin username:'))
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
-            email = input('Specify admin email:')
+            email = eval(input('Specify admin email:'))
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution.exchange'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'topic'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
         defaults = [
             ('auth_plugins',
              'egg:rhodecode-enterprise-ce#token,egg:rhodecode-enterprise-ce#rhodecode',
              'list'),
 
             ('auth_authtoken_enabled',
              'True',
              'bool'),
 
             ('auth_rhodecode_enabled',
              'True',
              'bool'),
         ]
         for k, v, t in defaults:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
-            path = input(
+            path = eval(input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
-            )
+            ))
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
 
             q = ('Given path %s is not writeable, do you want to '
                  'continue with read only mode ? [y/n]' % (path,))
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = ('Path looks like a symlink, RhodeCode Enterprise will store '
                  'given path as %s ? [y/n]') % (real_path,)
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
 
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
             ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
 
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
 
             # VCS Settings
             ('pr_merge_enabled', True, 'bool'),
             ('use_outdated_comments', True, 'bool'),
             ('diff_cache', True, 'bool'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`', username)
         user = UserModel().create_or_update(
-            username, password, email, firstname=u'RhodeCode', lastname=u'Admin',
+            username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
-                description=u'BUILTIN TOKEN')
+                description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
-                                            firstname=u'Anonymous',
-                                            lastname=u'User',
+                                            firstname='Anonymous',
+                                            lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
-        # controlls anonymous access
+        # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/diff_match_patch.py

@@ -1,2031 +1,2031 b''
 """Diff Match and Patch
 
 Copyright 2006 Google Inc.
 http://code.google.com/p/google-diff-match-patch/
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
 
 """Functions for diff, match and patch.
 
 Computes the difference between two texts to create a patch.
 Applies the patch onto another text, allowing for errors.
 """
 
 __author__ = "[e-mail] (Neil Fraser)"
 
 import math
 import re
 import sys
 import time
 import urllib.request, urllib.parse, urllib.error
 
 
 class diff_match_patch:
     """Class containing the diff, match and patch methods.
 
     Also contains the behaviour settings.
     """
 
     def __init__(self):
         """Inits a diff_match_patch object with default settings.
         Redefine these in your program to override the defaults.
         """
 
         # Number of seconds to map a diff before giving up (0 for infinity).
         self.Diff_Timeout = 1.0
         # Cost of an empty edit operation in terms of edit characters.
         self.Diff_EditCost = 4
         # At what point is no match declared (0.0 = perfection, 1.0 = very loose).
         self.Match_Threshold = 0.5
         # How far to search for a match (0 = exact location, 1000+ = broad match).
         # A match this many characters away from the expected location will add
         # 1.0 to the score (0.0 is a perfect match).
         self.Match_Distance = 1000
         # When deleting a large block of text (over ~64 characters), how close do
         # the contents have to be to match the expected contents. (0.0 = perfection,
         # 1.0 = very loose).  Note that Match_Threshold controls how closely the
         # end points of a delete need to match.
         self.Patch_DeleteThreshold = 0.5
         # Chunk size for context length.
         self.Patch_Margin = 4
 
         # The number of bits in an int.
         # Python has no maximum, thus to disable patch splitting set to 0.
         # However to avoid long patches in certain pathological cases, use 32.
         # Multiple short patches (using native ints) are much faster than long ones.
         self.Match_MaxBits = 32
 
     #  DIFF FUNCTIONS
 
     # The data structure representing a diff is an array of tuples:
     # [(DIFF_DELETE, "Hello"), (DIFF_INSERT, "Goodbye"), (DIFF_EQUAL, " world.")]
     # which means: delete "Hello", add "Goodbye" and keep " world."
     DIFF_DELETE = -1
     DIFF_INSERT = 1
     DIFF_EQUAL = 0
 
     def diff_main(self, text1, text2, checklines=True, deadline=None):
         """Find the differences between two texts.  Simplifies the problem by
           stripping any common prefix or suffix off the texts before diffing.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           checklines: Optional speedup flag.  If present and false, then don't run
             a line-level diff first to identify the changed areas.
             Defaults to true, which does a faster, slightly less optimal diff.
           deadline: Optional time when the diff should be complete by.  Used
             internally for recursive calls.  Users should set DiffTimeout instead.
 
         Returns:
           Array of changes.
         """
         # Set a deadline by which time the diff must be complete.
         if deadline is None:
             # Unlike in most languages, Python counts time in seconds.
             if self.Diff_Timeout <= 0:
                 deadline = sys.maxsize
             else:
                 deadline = time.time() + self.Diff_Timeout
 
         # Check for null inputs.
         if text1 is None or text2 is None:
             raise ValueError("Null inputs. (diff_main)")
 
         # Check for equality (speedup).
         if text1 == text2:
             if text1:
                 return [(self.DIFF_EQUAL, text1)]
             return []
 
         # Trim off common prefix (speedup).
         commonlength = self.diff_commonPrefix(text1, text2)
         commonprefix = text1[:commonlength]
         text1 = text1[commonlength:]
         text2 = text2[commonlength:]
 
         # Trim off common suffix (speedup).
         commonlength = self.diff_commonSuffix(text1, text2)
         if commonlength == 0:
             commonsuffix = ""
         else:
             commonsuffix = text1[-commonlength:]
             text1 = text1[:-commonlength]
             text2 = text2[:-commonlength]
 
         # Compute the diff on the middle block.
         diffs = self.diff_compute(text1, text2, checklines, deadline)
 
         # Restore the prefix and suffix.
         if commonprefix:
             diffs[:0] = [(self.DIFF_EQUAL, commonprefix)]
         if commonsuffix:
             diffs.append((self.DIFF_EQUAL, commonsuffix))
         self.diff_cleanupMerge(diffs)
         return diffs
 
     def diff_compute(self, text1, text2, checklines, deadline):
         """Find the differences between two texts.  Assumes that the texts do not
           have any common prefix or suffix.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           checklines: Speedup flag.  If false, then don't run a line-level diff
             first to identify the changed areas.
             If true, then run a faster, slightly less optimal diff.
           deadline: Time when the diff should be complete by.
 
         Returns:
           Array of changes.
         """
         if not text1:
             # Just add some text (speedup).
             return [(self.DIFF_INSERT, text2)]
 
         if not text2:
             # Just delete some text (speedup).
             return [(self.DIFF_DELETE, text1)]
 
         if len(text1) > len(text2):
             (longtext, shorttext) = (text1, text2)
         else:
             (shorttext, longtext) = (text1, text2)
         i = longtext.find(shorttext)
         if i != -1:
             # Shorter text is inside the longer text (speedup).
             diffs = [
                 (self.DIFF_INSERT, longtext[:i]),
                 (self.DIFF_EQUAL, shorttext),
                 (self.DIFF_INSERT, longtext[i + len(shorttext) :]),
             ]
             # Swap insertions for deletions if diff is reversed.
             if len(text1) > len(text2):
                 diffs[0] = (self.DIFF_DELETE, diffs[0][1])
                 diffs[2] = (self.DIFF_DELETE, diffs[2][1])
             return diffs
 
         if len(shorttext) == 1:
             # Single character string.
             # After the previous speedup, the character can't be an equality.
             return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
         # Check to see if the problem can be split in two.
         hm = self.diff_halfMatch(text1, text2)
         if hm:
             # A half-match was found, sort out the return data.
             (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
             # Send both pairs off for separate processing.
             diffs_a = self.diff_main(text1_a, text2_a, checklines, deadline)
             diffs_b = self.diff_main(text1_b, text2_b, checklines, deadline)
             # Merge the results.
             return diffs_a + [(self.DIFF_EQUAL, mid_common)] + diffs_b
 
         if checklines and len(text1) > 100 and len(text2) > 100:
             return self.diff_lineMode(text1, text2, deadline)
 
         return self.diff_bisect(text1, text2, deadline)
 
     def diff_lineMode(self, text1, text2, deadline):
         """Do a quick line-level diff on both strings, then rediff the parts for
           greater accuracy.
           This speedup can produce non-minimal diffs.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           deadline: Time when the diff should be complete by.
 
         Returns:
           Array of changes.
         """
 
         # Scan the text on a line-by-line basis first.
         (text1, text2, linearray) = self.diff_linesToChars(text1, text2)
 
         diffs = self.diff_main(text1, text2, False, deadline)
 
         # Convert the diff back to original text.
         self.diff_charsToLines(diffs, linearray)
         # Eliminate freak matches (e.g. blank lines)
         self.diff_cleanupSemantic(diffs)
 
         # Rediff any replacement blocks, this time character-by-character.
         # Add a dummy entry at the end.
         diffs.append((self.DIFF_EQUAL, ""))
         pointer = 0
         count_delete = 0
         count_insert = 0
         text_delete = ""
         text_insert = ""
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_INSERT:
                 count_insert += 1
                 text_insert += diffs[pointer][1]
             elif diffs[pointer][0] == self.DIFF_DELETE:
                 count_delete += 1
                 text_delete += diffs[pointer][1]
             elif diffs[pointer][0] == self.DIFF_EQUAL:
                 # Upon reaching an equality, check for prior redundancies.
                 if count_delete >= 1 and count_insert >= 1:
                     # Delete the offending records and add the merged ones.
                     a = self.diff_main(text_delete, text_insert, False, deadline)
                     diffs[pointer - count_delete - count_insert : pointer] = a
                     pointer = pointer - count_delete - count_insert + len(a)
                 count_insert = 0
                 count_delete = 0
                 text_delete = ""
                 text_insert = ""
 
             pointer += 1
 
         diffs.pop()  # Remove the dummy entry at the end.
 
         return diffs
 
     def diff_bisect(self, text1, text2, deadline):
         """Find the 'middle snake' of a diff, split the problem in two
           and return the recursively constructed diff.
           See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           deadline: Time at which to bail if not yet complete.
 
         Returns:
           Array of diff tuples.
         """
 
         # Cache the text lengths to prevent multiple calls.
         text1_length = len(text1)
         text2_length = len(text2)
         max_d = (text1_length + text2_length + 1) // 2
         v_offset = max_d
         v_length = 2 * max_d
         v1 = [-1] * v_length
         v1[v_offset + 1] = 0
         v2 = v1[:]
         delta = text1_length - text2_length
         # If the total number of characters is odd, then the front path will
         # collide with the reverse path.
         front = delta % 2 != 0
         # Offsets for start and end of k loop.
         # Prevents mapping of space beyond the grid.
         k1start = 0
         k1end = 0
         k2start = 0
         k2end = 0
         for d in range(max_d):
             # Bail out if deadline is reached.
             if time.time() > deadline:
                 break
 
             # Walk the front path one step.
             for k1 in range(-d + k1start, d + 1 - k1end, 2):
                 k1_offset = v_offset + k1
                 if k1 == -d or (k1 != d and v1[k1_offset - 1] < v1[k1_offset + 1]):
                     x1 = v1[k1_offset + 1]
                 else:
                     x1 = v1[k1_offset - 1] + 1
                 y1 = x1 - k1
                 while (
                     x1 < text1_length and y1 < text2_length and text1[x1] == text2[y1]
                 ):
                     x1 += 1
                     y1 += 1
                 v1[k1_offset] = x1
                 if x1 > text1_length:
                     # Ran off the right of the graph.
                     k1end += 2
                 elif y1 > text2_length:
                     # Ran off the bottom of the graph.
                     k1start += 2
                 elif front:
                     k2_offset = v_offset + delta - k1
                     if k2_offset >= 0 and k2_offset < v_length and v2[k2_offset] != -1:
                         # Mirror x2 onto top-left coordinate system.
                         x2 = text1_length - v2[k2_offset]
                         if x1 >= x2:
                             # Overlap detected.
                             return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
             # Walk the reverse path one step.
             for k2 in range(-d + k2start, d + 1 - k2end, 2):
                 k2_offset = v_offset + k2
                 if k2 == -d or (k2 != d and v2[k2_offset - 1] < v2[k2_offset + 1]):
                     x2 = v2[k2_offset + 1]
                 else:
                     x2 = v2[k2_offset - 1] + 1
                 y2 = x2 - k2
                 while (
                     x2 < text1_length
                     and y2 < text2_length
                     and text1[-x2 - 1] == text2[-y2 - 1]
                 ):
                     x2 += 1
                     y2 += 1
                 v2[k2_offset] = x2
                 if x2 > text1_length:
                     # Ran off the left of the graph.
                     k2end += 2
                 elif y2 > text2_length:
                     # Ran off the top of the graph.
                     k2start += 2
                 elif not front:
                     k1_offset = v_offset + delta - k2
                     if k1_offset >= 0 and k1_offset < v_length and v1[k1_offset] != -1:
                         x1 = v1[k1_offset]
                         y1 = v_offset + x1 - k1_offset
                         # Mirror x2 onto top-left coordinate system.
                         x2 = text1_length - x2
                         if x1 >= x2:
                             # Overlap detected.
                             return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
         # Diff took too long and hit the deadline or
         # number of diffs equals number of characters, no commonality at all.
         return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
     def diff_bisectSplit(self, text1, text2, x, y, deadline):
         """Given the location of the 'middle snake', split the diff in two parts
         and recurse.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           x: Index of split point in text1.
           y: Index of split point in text2.
           deadline: Time at which to bail if not yet complete.
 
         Returns:
           Array of diff tuples.
         """
         text1a = text1[:x]
         text2a = text2[:y]
         text1b = text1[x:]
         text2b = text2[y:]
 
         # Compute both diffs serially.
         diffs = self.diff_main(text1a, text2a, False, deadline)
         diffsb = self.diff_main(text1b, text2b, False, deadline)
 
         return diffs + diffsb
 
     def diff_linesToChars(self, text1, text2):
         """Split two texts into an array of strings.  Reduce the texts to a string
         of hashes where each Unicode character represents one line.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           Three element tuple, containing the encoded text1, the encoded text2 and
           the array of unique strings.  The zeroth element of the array of unique
           strings is intentionally blank.
         """
         lineArray = []  # e.g. lineArray[4] == "Hello\n"
         lineHash = {}  # e.g. lineHash["Hello\n"] == 4
 
         # "\x00" is a valid character, but various debuggers don't like it.
         # So we'll insert a junk entry to avoid generating a null character.
         lineArray.append("")
 
         def diff_linesToCharsMunge(text):
             """Split a text into an array of strings.  Reduce the texts to a string
             of hashes where each Unicode character represents one line.
             Modifies linearray and linehash through being a closure.
 
             Args:
               text: String to encode.
 
             Returns:
               Encoded string.
             """
             chars = []
             # Walk the text, pulling out a substring for each line.
             # text.split('\n') would would temporarily double our memory footprint.
             # Modifying text would create many large strings to garbage collect.
             lineStart = 0
             lineEnd = -1
             while lineEnd < len(text) - 1:
                 lineEnd = text.find("\n", lineStart)
                 if lineEnd == -1:
                     lineEnd = len(text) - 1
                 line = text[lineStart : lineEnd + 1]
                 lineStart = lineEnd + 1
 
                 if line in lineHash:
                     chars.append(chr(lineHash[line]))
                 else:
                     lineArray.append(line)
                     lineHash[line] = len(lineArray) - 1
                     chars.append(chr(len(lineArray) - 1))
             return "".join(chars)
 
         chars1 = diff_linesToCharsMunge(text1)
         chars2 = diff_linesToCharsMunge(text2)
         return (chars1, chars2, lineArray)
 
     def diff_charsToLines(self, diffs, lineArray):
         """Rehydrate the text in a diff from a string of line hashes to real lines
         of text.
 
         Args:
           diffs: Array of diff tuples.
           lineArray: Array of unique strings.
         """
         for x in range(len(diffs)):
             text = []
             for char in diffs[x][1]:
                 text.append(lineArray[ord(char)])
             diffs[x] = (diffs[x][0], "".join(text))
 
     def diff_commonPrefix(self, text1, text2):
         """Determine the common prefix of two strings.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           The number of characters common to the start of each string.
         """
         # Quick check for common null cases.
         if not text1 or not text2 or text1[0] != text2[0]:
             return 0
         # Binary search.
         # Performance analysis: http://neil.fraser.name/news/2007/10/09/
         pointermin = 0
         pointermax = min(len(text1), len(text2))
         pointermid = pointermax
         pointerstart = 0
         while pointermin < pointermid:
             if text1[pointerstart:pointermid] == text2[pointerstart:pointermid]:
                 pointermin = pointermid
                 pointerstart = pointermin
             else:
                 pointermax = pointermid
             pointermid = (pointermax - pointermin) // 2 + pointermin
         return pointermid
 
     def diff_commonSuffix(self, text1, text2):
         """Determine the common suffix of two strings.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           The number of characters common to the end of each string.
         """
         # Quick check for common null cases.
         if not text1 or not text2 or text1[-1] != text2[-1]:
             return 0
         # Binary search.
         # Performance analysis: http://neil.fraser.name/news/2007/10/09/
         pointermin = 0
         pointermax = min(len(text1), len(text2))
         pointermid = pointermax
         pointerend = 0
         while pointermin < pointermid:
             if (
                 text1[-pointermid : len(text1) - pointerend]
                 == text2[-pointermid : len(text2) - pointerend]
             ):
                 pointermin = pointermid
                 pointerend = pointermin
             else:
                 pointermax = pointermid
             pointermid = (pointermax - pointermin) // 2 + pointermin
         return pointermid
 
     def diff_commonOverlap(self, text1, text2):
         """Determine if the suffix of one string is the prefix of another.
 
         Args:
           text1 First string.
           text2 Second string.
 
         Returns:
           The number of characters common to the end of the first
           string and the start of the second string.
         """
         # Cache the text lengths to prevent multiple calls.
         text1_length = len(text1)
         text2_length = len(text2)
         # Eliminate the null case.
         if text1_length == 0 or text2_length == 0:
             return 0
         # Truncate the longer string.
         if text1_length > text2_length:
             text1 = text1[-text2_length:]
         elif text1_length < text2_length:
             text2 = text2[:text1_length]
         text_length = min(text1_length, text2_length)
         # Quick check for the worst case.
         if text1 == text2:
             return text_length
 
         # Start by looking for a single character match
         # and increase length until no match is found.
         # Performance analysis: http://neil.fraser.name/news/2010/11/04/
         best = 0
         length = 1
         while True:
             pattern = text1[-length:]
             found = text2.find(pattern)
             if found == -1:
                 return best
             length += found
             if found == 0 or text1[-length:] == text2[:length]:
                 best = length
                 length += 1
 
     def diff_halfMatch(self, text1, text2):
         """Do the two texts share a substring which is at least half the length of
         the longer text?
         This speedup can produce non-minimal diffs.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           Five element Array, containing the prefix of text1, the suffix of text1,
           the prefix of text2, the suffix of text2 and the common middle.  Or None
           if there was no match.
         """
         if self.Diff_Timeout <= 0:
             # Don't risk returning a non-optimal diff if we have unlimited time.
             return None
         if len(text1) > len(text2):
             (longtext, shorttext) = (text1, text2)
         else:
             (shorttext, longtext) = (text1, text2)
         if len(longtext) < 4 or len(shorttext) * 2 < len(longtext):
             return None  # Pointless.
 
         def diff_halfMatchI(longtext, shorttext, i):
             """Does a substring of shorttext exist within longtext such that the
             substring is at least half the length of longtext?
             Closure, but does not reference any external variables.
 
             Args:
               longtext: Longer string.
               shorttext: Shorter string.
               i: Start index of quarter length substring within longtext.
 
             Returns:
               Five element Array, containing the prefix of longtext, the suffix of
               longtext, the prefix of shorttext, the suffix of shorttext and the
               common middle.  Or None if there was no match.
             """
             seed = longtext[i : i + len(longtext) // 4]
             best_common = ""
             j = shorttext.find(seed)
             while j != -1:
                 prefixLength = self.diff_commonPrefix(longtext[i:], shorttext[j:])
                 suffixLength = self.diff_commonSuffix(longtext[:i], shorttext[:j])
                 if len(best_common) < suffixLength + prefixLength:
                     best_common = (
                         shorttext[j - suffixLength : j]
                         + shorttext[j : j + prefixLength]
                     )
                     best_longtext_a = longtext[: i - suffixLength]
                     best_longtext_b = longtext[i + prefixLength :]
                     best_shorttext_a = shorttext[: j - suffixLength]
                     best_shorttext_b = shorttext[j + prefixLength :]
                 j = shorttext.find(seed, j + 1)
 
             if len(best_common) * 2 >= len(longtext):
                 return (
                     best_longtext_a,
                     best_longtext_b,
                     best_shorttext_a,
                     best_shorttext_b,
                     best_common,
                 )
             else:
                 return None
 
         # First check if the second quarter is the seed for a half-match.
         hm1 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 3) // 4)
         # Check again based on the third quarter.
         hm2 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 1) // 2)
         if not hm1 and not hm2:
             return None
         elif not hm2:
             hm = hm1
         elif not hm1:
             hm = hm2
         else:
             # Both matched.  Select the longest.
             if len(hm1[4]) > len(hm2[4]):
                 hm = hm1
             else:
                 hm = hm2
 
         # A half-match was found, sort out the return data.
         if len(text1) > len(text2):
             (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
         else:
             (text2_a, text2_b, text1_a, text1_b, mid_common) = hm
         return (text1_a, text1_b, text2_a, text2_b, mid_common)
 
     def diff_cleanupSemantic(self, diffs):
         """Reduce the number of edits by eliminating semantically trivial
         equalities.
 
         Args:
           diffs: Array of diff tuples.
         """
         changes = False
         equalities = []  # Stack of indices where equalities are found.
         lastequality = None  # Always equal to diffs[equalities[-1]][1]
         pointer = 0  # Index of current position.
         # Number of chars that changed prior to the equality.
         length_insertions1, length_deletions1 = 0, 0
         # Number of chars that changed after the equality.
         length_insertions2, length_deletions2 = 0, 0
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
                 equalities.append(pointer)
                 length_insertions1, length_insertions2 = length_insertions2, 0
                 length_deletions1, length_deletions2 = length_deletions2, 0
                 lastequality = diffs[pointer][1]
             else:  # An insertion or deletion.
                 if diffs[pointer][0] == self.DIFF_INSERT:
                     length_insertions2 += len(diffs[pointer][1])
                 else:
                     length_deletions2 += len(diffs[pointer][1])
                 # Eliminate an equality that is smaller or equal to the edits on both
                 # sides of it.
                 if (
                     lastequality
                     and (
                         len(lastequality) <= max(length_insertions1, length_deletions1)
                     )
                     and (
                         len(lastequality) <= max(length_insertions2, length_deletions2)
                     )
                 ):
                     # Duplicate record.
                     diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
                     # Change second copy to insert.
                     diffs[equalities[-1] + 1] = (
                         self.DIFF_INSERT,
                         diffs[equalities[-1] + 1][1],
                     )
                     # Throw away the equality we just deleted.
                     equalities.pop()
                     # Throw away the previous equality (it needs to be reevaluated).
                     if len(equalities):
                         equalities.pop()
                     if len(equalities):
                         pointer = equalities[-1]
                     else:
                         pointer = -1
                     # Reset the counters.
                     length_insertions1, length_deletions1 = 0, 0
                     length_insertions2, length_deletions2 = 0, 0
                     lastequality = None
                     changes = True
             pointer += 1
 
         # Normalize the diff.
         if changes:
             self.diff_cleanupMerge(diffs)
         self.diff_cleanupSemanticLossless(diffs)
 
         # Find any overlaps between deletions and insertions.
         # e.g: <del>abcxxx</del><ins>xxxdef</ins>
         #   -> <del>abc</del>xxx<ins>def</ins>
         # e.g: <del>xxxabc</del><ins>defxxx</ins>
         #   -> <ins>def</ins>xxx<del>abc</del>
         # Only extract an overlap if it is as big as the edit ahead or behind it.
         pointer = 1
         while pointer < len(diffs):
             if (
                 diffs[pointer - 1][0] == self.DIFF_DELETE
                 and diffs[pointer][0] == self.DIFF_INSERT
             ):
                 deletion = diffs[pointer - 1][1]
                 insertion = diffs[pointer][1]
                 overlap_length1 = self.diff_commonOverlap(deletion, insertion)
                 overlap_length2 = self.diff_commonOverlap(insertion, deletion)
                 if overlap_length1 >= overlap_length2:
                     if (
                         overlap_length1 >= len(deletion) / 2.0
                         or overlap_length1 >= len(insertion) / 2.0
                     ):
                         # Overlap found.  Insert an equality and trim the surrounding edits.
                         diffs.insert(
                             pointer, (self.DIFF_EQUAL, insertion[:overlap_length1])
                         )
                         diffs[pointer - 1] = (
                             self.DIFF_DELETE,
                             deletion[: len(deletion) - overlap_length1],
                         )
                         diffs[pointer + 1] = (
                             self.DIFF_INSERT,
                             insertion[overlap_length1:],
                         )
                         pointer += 1
                 else:
                     if (
                         overlap_length2 >= len(deletion) / 2.0
                         or overlap_length2 >= len(insertion) / 2.0
                     ):
                         # Reverse overlap found.
                         # Insert an equality and swap and trim the surrounding edits.
                         diffs.insert(
                             pointer, (self.DIFF_EQUAL, deletion[:overlap_length2])
                         )
                         diffs[pointer - 1] = (
                             self.DIFF_INSERT,
                             insertion[: len(insertion) - overlap_length2],
                         )
                         diffs[pointer + 1] = (
                             self.DIFF_DELETE,
                             deletion[overlap_length2:],
                         )
                         pointer += 1
                 pointer += 1
             pointer += 1
 
     def diff_cleanupSemanticLossless(self, diffs):
         """Look for single edits surrounded on both sides by equalities
         which can be shifted sideways to align the edit to a word boundary.
         e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.
 
         Args:
           diffs: Array of diff tuples.
         """
 
         def diff_cleanupSemanticScore(one, two):
             """Given two strings, compute a score representing whether the
             internal boundary falls on logical boundaries.
             Scores range from 6 (best) to 0 (worst).
             Closure, but does not reference any external variables.
 
             Args:
               one: First string.
               two: Second string.
 
             Returns:
               The score.
             """
             if not one or not two:
                 # Edges are the best.
                 return 6
 
             # Each port of this function behaves slightly differently due to
             # subtle differences in each language's definition of things like
             # 'whitespace'.  Since this function's purpose is largely cosmetic,
             # the choice has been made to use each language's native features
             # rather than force total conformity.
             char1 = one[-1]
             char2 = two[0]
             nonAlphaNumeric1 = not char1.isalnum()
             nonAlphaNumeric2 = not char2.isalnum()
             whitespace1 = nonAlphaNumeric1 and char1.isspace()
             whitespace2 = nonAlphaNumeric2 and char2.isspace()
             lineBreak1 = whitespace1 and (char1 == "\r" or char1 == "\n")
             lineBreak2 = whitespace2 and (char2 == "\r" or char2 == "\n")
             blankLine1 = lineBreak1 and self.BLANKLINEEND.search(one)
             blankLine2 = lineBreak2 and self.BLANKLINESTART.match(two)
 
             if blankLine1 or blankLine2:
                 # Five points for blank lines.
                 return 5
             elif lineBreak1 or lineBreak2:
                 # Four points for line breaks.
                 return 4
             elif nonAlphaNumeric1 and not whitespace1 and whitespace2:
                 # Three points for end of sentences.
                 return 3
             elif whitespace1 or whitespace2:
                 # Two points for whitespace.
                 return 2
             elif nonAlphaNumeric1 or nonAlphaNumeric2:
                 # One point for non-alphanumeric.
                 return 1
             return 0
 
         pointer = 1
         # Intentionally ignore the first and last element (don't need checking).
         while pointer < len(diffs) - 1:
             if (
                 diffs[pointer - 1][0] == self.DIFF_EQUAL
                 and diffs[pointer + 1][0] == self.DIFF_EQUAL
             ):
                 # This is a single edit surrounded by equalities.
                 equality1 = diffs[pointer - 1][1]
                 edit = diffs[pointer][1]
                 equality2 = diffs[pointer + 1][1]
 
                 # First, shift the edit as far left as possible.
                 commonOffset = self.diff_commonSuffix(equality1, edit)
                 if commonOffset:
                     commonString = edit[-commonOffset:]
                     equality1 = equality1[:-commonOffset]
                     edit = commonString + edit[:-commonOffset]
                     equality2 = commonString + equality2
 
                 # Second, step character by character right, looking for the best fit.
                 bestEquality1 = equality1
                 bestEdit = edit
                 bestEquality2 = equality2
                 bestScore = diff_cleanupSemanticScore(
                     equality1, edit
                 ) + diff_cleanupSemanticScore(edit, equality2)
                 while edit and equality2 and edit[0] == equality2[0]:
                     equality1 += edit[0]
                     edit = edit[1:] + equality2[0]
                     equality2 = equality2[1:]
                     score = diff_cleanupSemanticScore(
                         equality1, edit
                     ) + diff_cleanupSemanticScore(edit, equality2)
                     # The >= encourages trailing rather than leading whitespace on edits.
                     if score >= bestScore:
                         bestScore = score
                         bestEquality1 = equality1
                         bestEdit = edit
                         bestEquality2 = equality2
 
                 if diffs[pointer - 1][1] != bestEquality1:
                     # We have an improvement, save it back to the diff.
                     if bestEquality1:
                         diffs[pointer - 1] = (diffs[pointer - 1][0], bestEquality1)
                     else:
                         del diffs[pointer - 1]
                         pointer -= 1
                     diffs[pointer] = (diffs[pointer][0], bestEdit)
                     if bestEquality2:
                         diffs[pointer + 1] = (diffs[pointer + 1][0], bestEquality2)
                     else:
                         del diffs[pointer + 1]
                         pointer -= 1
             pointer += 1
 
     # Define some regex patterns for matching boundaries.
     BLANKLINEEND = re.compile(r"\n\r?\n$")
     BLANKLINESTART = re.compile(r"^\r?\n\r?\n")
 
     def diff_cleanupEfficiency(self, diffs):
         """Reduce the number of edits by eliminating operationally trivial
         equalities.
 
         Args:
           diffs: Array of diff tuples.
         """
         changes = False
         equalities = []  # Stack of indices where equalities are found.
         lastequality = None  # Always equal to diffs[equalities[-1]][1]
         pointer = 0  # Index of current position.
         pre_ins = False  # Is there an insertion operation before the last equality.
         pre_del = False  # Is there a deletion operation before the last equality.
         post_ins = False  # Is there an insertion operation after the last equality.
         post_del = False  # Is there a deletion operation after the last equality.
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
                 if len(diffs[pointer][1]) < self.Diff_EditCost and (
                     post_ins or post_del
                 ):
                     # Candidate found.
                     equalities.append(pointer)
                     pre_ins = post_ins
                     pre_del = post_del
                     lastequality = diffs[pointer][1]
                 else:
                     # Not a candidate, and can never become one.
                     equalities = []
                     lastequality = None
 
                 post_ins = post_del = False
             else:  # An insertion or deletion.
                 if diffs[pointer][0] == self.DIFF_DELETE:
                     post_del = True
                 else:
                     post_ins = True
 
                 # Five types to be split:
                 # <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
                 # <ins>A</ins>X<ins>C</ins><del>D</del>
                 # <ins>A</ins><del>B</del>X<ins>C</ins>
                 # <ins>A</del>X<ins>C</ins><del>D</del>
                 # <ins>A</ins><del>B</del>X<del>C</del>
 
                 if lastequality and (
                     (pre_ins and pre_del and post_ins and post_del)
                     or (
                         (len(lastequality) < self.Diff_EditCost / 2)
                         and (pre_ins + pre_del + post_ins + post_del) == 3
                     )
                 ):
                     # Duplicate record.
                     diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
                     # Change second copy to insert.
                     diffs[equalities[-1] + 1] = (
                         self.DIFF_INSERT,
                         diffs[equalities[-1] + 1][1],
                     )
                     equalities.pop()  # Throw away the equality we just deleted.
                     lastequality = None
                     if pre_ins and pre_del:
                         # No changes made which could affect previous entry, keep going.
                         post_ins = post_del = True
                         equalities = []
                     else:
                         if len(equalities):
                             equalities.pop()  # Throw away the previous equality.
                         if len(equalities):
                             pointer = equalities[-1]
                         else:
                             pointer = -1
                         post_ins = post_del = False
                     changes = True
             pointer += 1
 
         if changes:
             self.diff_cleanupMerge(diffs)
 
     def diff_cleanupMerge(self, diffs):
         """Reorder and merge like edit sections.  Merge equalities.
         Any edit section can move as long as it doesn't cross an equality.
 
         Args:
           diffs: Array of diff tuples.
         """
         diffs.append((self.DIFF_EQUAL, ""))  # Add a dummy entry at the end.
         pointer = 0
         count_delete = 0
         count_insert = 0
         text_delete = ""
         text_insert = ""
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_INSERT:
                 count_insert += 1
                 text_insert += diffs[pointer][1]
                 pointer += 1
             elif diffs[pointer][0] == self.DIFF_DELETE:
                 count_delete += 1
                 text_delete += diffs[pointer][1]
                 pointer += 1
             elif diffs[pointer][0] == self.DIFF_EQUAL:
                 # Upon reaching an equality, check for prior redundancies.
                 if count_delete + count_insert > 1:
                     if count_delete != 0 and count_insert != 0:
                         # Factor out any common prefixies.
                         commonlength = self.diff_commonPrefix(text_insert, text_delete)
                         if commonlength != 0:
                             x = pointer - count_delete - count_insert - 1
                             if x >= 0 and diffs[x][0] == self.DIFF_EQUAL:
                                 diffs[x] = (
                                     diffs[x][0],
                                     diffs[x][1] + text_insert[:commonlength],
                                 )
                             else:
                                 diffs.insert(
                                     0, (self.DIFF_EQUAL, text_insert[:commonlength])
                                 )
                                 pointer += 1
                             text_insert = text_insert[commonlength:]
                             text_delete = text_delete[commonlength:]
                         # Factor out any common suffixies.
                         commonlength = self.diff_commonSuffix(text_insert, text_delete)
                         if commonlength != 0:
                             diffs[pointer] = (
                                 diffs[pointer][0],
                                 text_insert[-commonlength:] + diffs[pointer][1],
                             )
                             text_insert = text_insert[:-commonlength]
                             text_delete = text_delete[:-commonlength]
                     # Delete the offending records and add the merged ones.
                     if count_delete == 0:
                         diffs[pointer - count_insert : pointer] = [
                             (self.DIFF_INSERT, text_insert)
                         ]
                     elif count_insert == 0:
                         diffs[pointer - count_delete : pointer] = [
                             (self.DIFF_DELETE, text_delete)
                         ]
                     else:
                         diffs[pointer - count_delete - count_insert : pointer] = [
                             (self.DIFF_DELETE, text_delete),
                             (self.DIFF_INSERT, text_insert),
                         ]
                     pointer = pointer - count_delete - count_insert + 1
                     if count_delete != 0:
                         pointer += 1
                     if count_insert != 0:
                         pointer += 1
                 elif pointer != 0 and diffs[pointer - 1][0] == self.DIFF_EQUAL:
                     # Merge this equality with the previous one.
                     diffs[pointer - 1] = (
                         diffs[pointer - 1][0],
                         diffs[pointer - 1][1] + diffs[pointer][1],
                     )
                     del diffs[pointer]
                 else:
                     pointer += 1
 
                 count_insert = 0
                 count_delete = 0
                 text_delete = ""
                 text_insert = ""
 
         if diffs[-1][1] == "":
             diffs.pop()  # Remove the dummy entry at the end.
 
         # Second pass: look for single edits surrounded on both sides by equalities
         # which can be shifted sideways to eliminate an equality.
         # e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
         changes = False
         pointer = 1
         # Intentionally ignore the first and last element (don't need checking).
         while pointer < len(diffs) - 1:
             if (
                 diffs[pointer - 1][0] == self.DIFF_EQUAL
                 and diffs[pointer + 1][0] == self.DIFF_EQUAL
             ):
                 # This is a single edit surrounded by equalities.
                 if diffs[pointer][1].endswith(diffs[pointer - 1][1]):
                     # Shift the edit over the previous equality.
                     diffs[pointer] = (
                         diffs[pointer][0],
                         diffs[pointer - 1][1]
                         + diffs[pointer][1][: -len(diffs[pointer - 1][1])],
                     )
                     diffs[pointer + 1] = (
                         diffs[pointer + 1][0],
                         diffs[pointer - 1][1] + diffs[pointer + 1][1],
                     )
                     del diffs[pointer - 1]
                     changes = True
                 elif diffs[pointer][1].startswith(diffs[pointer + 1][1]):
                     # Shift the edit over the next equality.
                     diffs[pointer - 1] = (
                         diffs[pointer - 1][0],
                         diffs[pointer - 1][1] + diffs[pointer + 1][1],
                     )
                     diffs[pointer] = (
                         diffs[pointer][0],
                         diffs[pointer][1][len(diffs[pointer + 1][1]) :]
                         + diffs[pointer + 1][1],
                     )
                     del diffs[pointer + 1]
                     changes = True
             pointer += 1
 
         # If shifts were made, the diff needs reordering and another shift sweep.
         if changes:
             self.diff_cleanupMerge(diffs)
 
     def diff_xIndex(self, diffs, loc):
         """loc is a location in text1, compute and return the equivalent location
         in text2.  e.g. "The cat" vs "The big cat", 1->1, 5->8
 
         Args:
           diffs: Array of diff tuples.
           loc: Location within text1.
 
         Returns:
           Location within text2.
         """
         chars1 = 0
         chars2 = 0
         last_chars1 = 0
         last_chars2 = 0
         for x in range(len(diffs)):
             (op, text) = diffs[x]
             if op != self.DIFF_INSERT:  # Equality or deletion.
                 chars1 += len(text)
             if op != self.DIFF_DELETE:  # Equality or insertion.
                 chars2 += len(text)
             if chars1 > loc:  # Overshot the location.
                 break
             last_chars1 = chars1
             last_chars2 = chars2
 
         if len(diffs) != x and diffs[x][0] == self.DIFF_DELETE:
             # The location was deleted.
             return last_chars2
         # Add the remaining len(character).
         return last_chars2 + (loc - last_chars1)
 
     def diff_prettyHtml(self, diffs):
         """Convert a diff array into a pretty HTML report.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           HTML representation.
         """
         html = []
         for op, data in diffs:
             text = (
                 data.replace("&", "&amp;")
                 .replace("<", "&lt;")
                 .replace(">", "&gt;")
                 .replace("\n", "&para;<br>")
             )
             if op == self.DIFF_INSERT:
                 html.append('<ins style="background:#e6ffe6;">%s</ins>' % text)
             elif op == self.DIFF_DELETE:
                 html.append('<del style="background:#ffe6e6;">%s</del>' % text)
             elif op == self.DIFF_EQUAL:
                 html.append("<span>%s</span>" % text)
         return "".join(html)
 
     def diff_text1(self, diffs):
         """Compute and return the source text (all equalities and deletions).
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Source text.
         """
         text = []
         for op, data in diffs:
             if op != self.DIFF_INSERT:
                 text.append(data)
         return "".join(text)
 
     def diff_text2(self, diffs):
         """Compute and return the destination text (all equalities and insertions).
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Destination text.
         """
         text = []
         for op, data in diffs:
             if op != self.DIFF_DELETE:
                 text.append(data)
         return "".join(text)
 
     def diff_levenshtein(self, diffs):
         """Compute the Levenshtein distance; the number of inserted, deleted or
         substituted characters.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Number of changes.
         """
         levenshtein = 0
         insertions = 0
         deletions = 0
         for op, data in diffs:
             if op == self.DIFF_INSERT:
                 insertions += len(data)
             elif op == self.DIFF_DELETE:
                 deletions += len(data)
             elif op == self.DIFF_EQUAL:
                 # A deletion and an insertion is one substitution.
                 levenshtein += max(insertions, deletions)
                 insertions = 0
                 deletions = 0
         levenshtein += max(insertions, deletions)
         return levenshtein
 
     def diff_toDelta(self, diffs):
         """Crush the diff into an encoded string which describes the operations
         required to transform text1 into text2.
         E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
         Operations are tab-separated.  Inserted text is escaped using %xx notation.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Delta text.
         """
         text = []
         for op, data in diffs:
             if op == self.DIFF_INSERT:
                 # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
                 data = data.encode("utf-8")
                 text.append("+" + urllib.parse.quote(data, "!~*'();/?:@&=+$,# "))
             elif op == self.DIFF_DELETE:
                 text.append("-%d" % len(data))
             elif op == self.DIFF_EQUAL:
                 text.append("=%d" % len(data))
         return "\t".join(text)
 
     def diff_fromDelta(self, text1, delta):
         """Given the original text1, and an encoded string which describes the
         operations required to transform text1 into text2, compute the full diff.
 
         Args:
           text1: Source string for the diff.
           delta: Delta text.
 
         Returns:
           Array of diff tuples.
 
         Raises:
           ValueError: If invalid input.
         """
         if type(delta) == str:
             # Deltas should be composed of a subset of ascii chars, Unicode not
             # required.  If this encode raises UnicodeEncodeError, delta is invalid.
             delta = delta.encode("ascii")
         diffs = []
         pointer = 0  # Cursor in text1
         tokens = delta.split("\t")
         for token in tokens:
             if token == "":
                 # Blank tokens are ok (from a trailing \t).
                 continue
             # Each token begins with a one character parameter which specifies the
             # operation of this token (delete, insert, equality).
             param = token[1:]
             if token[0] == "+":
                 param = urllib.parse.unquote(param)
                 diffs.append((self.DIFF_INSERT, param))
             elif token[0] == "-" or token[0] == "=":
                 try:
                     n = int(param)
                 except ValueError:
                     raise ValueError("Invalid number in diff_fromDelta: " + param)
                 if n < 0:
                     raise ValueError("Negative number in diff_fromDelta: " + param)
                 text = text1[pointer : pointer + n]
                 pointer += n
                 if token[0] == "=":
                     diffs.append((self.DIFF_EQUAL, text))
                 else:
                     diffs.append((self.DIFF_DELETE, text))
             else:
                 # Anything else is an error.
                 raise ValueError(
                     "Invalid diff operation in diff_fromDelta: " + token[0]
                 )
         if pointer != len(text1):
             raise ValueError(
                 "Delta length (%d) does not equal source text length (%d)."
                 % (pointer, len(text1))
             )
         return diffs
 
     #  MATCH FUNCTIONS
 
     def match_main(self, text, pattern, loc):
         """Locate the best instance of 'pattern' in 'text' near 'loc'.
 
         Args:
           text: The text to search.
           pattern: The pattern to search for.
           loc: The location to search around.
 
         Returns:
           Best match index or -1.
         """
         # Check for null inputs.
         if text is None or pattern is None:
             raise ValueError("Null inputs. (match_main)")
 
         loc = max(0, min(loc, len(text)))
         if text == pattern:
             # Shortcut (potentially not guaranteed by the algorithm)
             return 0
         elif not text:
             # Nothing to match.
             return -1
         elif text[loc : loc + len(pattern)] == pattern:
             # Perfect match at the perfect spot!  (Includes case of null pattern)
             return loc
         else:
             # Do a fuzzy compare.
             match = self.match_bitap(text, pattern, loc)
             return match
 
     def match_bitap(self, text, pattern, loc):
         """Locate the best instance of 'pattern' in 'text' near 'loc' using the
         Bitap algorithm.
 
         Args:
           text: The text to search.
           pattern: The pattern to search for.
           loc: The location to search around.
 
         Returns:
           Best match index or -1.
         """
         # Python doesn't have a maxint limit, so ignore this check.
         # if self.Match_MaxBits != 0 and len(pattern) > self.Match_MaxBits:
         #  raise ValueError("Pattern too long for this application.")
 
         # Initialise the alphabet.
         s = self.match_alphabet(pattern)
 
         def match_bitapScore(e, x):
             """Compute and return the score for a match with e errors and x location.
             Accesses loc and pattern through being a closure.
 
             Args:
               e: Number of errors in match.
               x: Location of match.
 
             Returns:
               Overall score for match (0.0 = good, 1.0 = bad).
             """
             accuracy = float(e) / len(pattern)
             proximity = abs(loc - x)
             if not self.Match_Distance:
                 # Dodge divide by zero error.
                 return proximity and 1.0 or accuracy
             return accuracy + (proximity / float(self.Match_Distance))
 
         # Highest score beyond which we give up.
         score_threshold = self.Match_Threshold
         # Is there a nearby exact match? (speedup)
         best_loc = text.find(pattern, loc)
         if best_loc != -1:
             score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
             # What about in the other direction? (speedup)
             best_loc = text.rfind(pattern, loc + len(pattern))
             if best_loc != -1:
                 score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
 
         # Initialise the bit arrays.
         matchmask = 1 << (len(pattern) - 1)
         best_loc = -1
 
         bin_max = len(pattern) + len(text)
         # Empty initialization added to appease pychecker.
         last_rd = None
         for d in range(len(pattern)):
             # Scan for the best match each iteration allows for one more error.
             # Run a binary search to determine how far from 'loc' we can stray at
             # this error level.
             bin_min = 0
             bin_mid = bin_max
             while bin_min < bin_mid:
                 if match_bitapScore(d, loc + bin_mid) <= score_threshold:
                     bin_min = bin_mid
                 else:
                     bin_max = bin_mid
                 bin_mid = (bin_max - bin_min) // 2 + bin_min
 
             # Use the result from this iteration as the maximum for the next.
             bin_max = bin_mid
             start = max(1, loc - bin_mid + 1)
             finish = min(loc + bin_mid, len(text)) + len(pattern)
 
             rd = [0] * (finish + 2)
             rd[finish + 1] = (1 << d) - 1
             for j in range(finish, start - 1, -1):
                 if len(text) <= j - 1:
                     # Out of range.
                     charMatch = 0
                 else:
                     charMatch = s.get(text[j - 1], 0)
                 if d == 0:  # First pass: exact match.
                     rd[j] = ((rd[j + 1] << 1) | 1) & charMatch
                 else:  # Subsequent passes: fuzzy match.
                     rd[j] = (
                         (((rd[j + 1] << 1) | 1) & charMatch)
                         | (((last_rd[j + 1] | last_rd[j]) << 1) | 1)
                         | last_rd[j + 1]
                     )
                 if rd[j] & matchmask:
                     score = match_bitapScore(d, j - 1)
                     # This match will almost certainly be better than any existing match.
                     # But check anyway.
                     if score <= score_threshold:
                         # Told you so.
                         score_threshold = score
                         best_loc = j - 1
                         if best_loc > loc:
                             # When passing loc, don't exceed our current distance from loc.
                             start = max(1, 2 * loc - best_loc)
                         else:
                             # Already passed loc, downhill from here on in.
                             break
             # No hope for a (better) match at greater error levels.
             if match_bitapScore(d + 1, loc) > score_threshold:
                 break
             last_rd = rd
         return best_loc
 
     def match_alphabet(self, pattern):
         """Initialise the alphabet for the Bitap algorithm.
 
         Args:
           pattern: The text to encode.
 
         Returns:
           Hash of character locations.
         """
         s = {}
         for char in pattern:
             s[char] = 0
         for i in range(len(pattern)):
             s[pattern[i]] |= 1 << (len(pattern) - i - 1)
         return s
 
     #  PATCH FUNCTIONS
 
     def patch_addContext(self, patch, text):
         """Increase the context until it is unique,
         but don't let the pattern expand beyond Match_MaxBits.
 
         Args:
           patch: The patch to grow.
           text: Source text.
         """
         if len(text) == 0:
             return
         pattern = text[patch.start2 : patch.start2 + patch.length1]
         padding = 0
 
         # Look for the first and last matches of pattern in text.  If two different
         # matches are found, increase the pattern length.
         while text.find(pattern) != text.rfind(pattern) and (
             self.Match_MaxBits == 0
             or len(pattern) < self.Match_MaxBits - self.Patch_Margin - self.Patch_Margin
         ):
             padding += self.Patch_Margin
             pattern = text[
                 max(0, patch.start2 - padding) : patch.start2 + patch.length1 + padding
             ]
         # Add one chunk for good luck.
         padding += self.Patch_Margin
 
         # Add the prefix.
         prefix = text[max(0, patch.start2 - padding) : patch.start2]
         if prefix:
             patch.diffs[:0] = [(self.DIFF_EQUAL, prefix)]
         # Add the suffix.
         suffix = text[
             patch.start2 + patch.length1 : patch.start2 + patch.length1 + padding
         ]
         if suffix:
             patch.diffs.append((self.DIFF_EQUAL, suffix))
 
         # Roll back the start points.
         patch.start1 -= len(prefix)
         patch.start2 -= len(prefix)
         # Extend lengths.
         patch.length1 += len(prefix) + len(suffix)
         patch.length2 += len(prefix) + len(suffix)
 
     def patch_make(self, a, b=None, c=None):
         """Compute a list of patches to turn text1 into text2.
         Use diffs if provided, otherwise compute it ourselves.
         There are four ways to call this function, depending on what data is
         available to the caller:
         Method 1:
         a = text1, b = text2
         Method 2:
         a = diffs
         Method 3 (optimal):
         a = text1, b = diffs
         Method 4 (deprecated, use method 3):
         a = text1, b = text2, c = diffs
 
         Args:
           a: text1 (methods 1,3,4) or Array of diff tuples for text1 to
               text2 (method 2).
           b: text2 (methods 1,4) or Array of diff tuples for text1 to
               text2 (method 3) or undefined (method 2).
           c: Array of diff tuples for text1 to text2 (method 4) or
               undefined (methods 1,2,3).
 
         Returns:
           Array of Patch objects.
         """
         text1 = None
         diffs = None
         # Note that texts may arrive as 'str' or 'unicode'.
         if isinstance(a, str) and isinstance(b, str) and c is None:
             # Method 1: text1, text2
             # Compute diffs from text1 and text2.
             text1 = a
             diffs = self.diff_main(text1, b, True)
             if len(diffs) > 2:
                 self.diff_cleanupSemantic(diffs)
                 self.diff_cleanupEfficiency(diffs)
         elif isinstance(a, list) and b is None and c is None:
             # Method 2: diffs
             # Compute text1 from diffs.
             diffs = a
             text1 = self.diff_text1(diffs)
         elif isinstance(a, str) and isinstance(b, list) and c is None:
             # Method 3: text1, diffs
             text1 = a
             diffs = b
         elif isinstance(a, str) and isinstance(b, str) and isinstance(c, list):
             # Method 4: text1, text2, diffs
             # text2 is not used.
             text1 = a
             diffs = c
         else:
             raise ValueError("Unknown call format to patch_make.")
 
         if not diffs:
             return []  # Get rid of the None case.
         patches = []
         patch = patch_obj()
         char_count1 = 0  # Number of characters into the text1 string.
         char_count2 = 0  # Number of characters into the text2 string.
         prepatch_text = text1  # Recreate the patches to determine context info.
         postpatch_text = text1
         for x in range(len(diffs)):
             (diff_type, diff_text) = diffs[x]
             if len(patch.diffs) == 0 and diff_type != self.DIFF_EQUAL:
                 # A new patch starts here.
                 patch.start1 = char_count1
                 patch.start2 = char_count2
             if diff_type == self.DIFF_INSERT:
                 # Insertion
                 patch.diffs.append(diffs[x])
                 patch.length2 += len(diff_text)
                 postpatch_text = (
                     postpatch_text[:char_count2]
                     + diff_text
                     + postpatch_text[char_count2:]
                 )
             elif diff_type == self.DIFF_DELETE:
                 # Deletion.
                 patch.length1 += len(diff_text)
                 patch.diffs.append(diffs[x])
                 postpatch_text = (
                     postpatch_text[:char_count2]
                     + postpatch_text[char_count2 + len(diff_text) :]
                 )
             elif (
                 diff_type == self.DIFF_EQUAL
                 and len(diff_text) <= 2 * self.Patch_Margin
                 and len(patch.diffs) != 0
                 and len(diffs) != x + 1
             ):
                 # Small equality inside a patch.
                 patch.diffs.append(diffs[x])
                 patch.length1 += len(diff_text)
                 patch.length2 += len(diff_text)
 
             if diff_type == self.DIFF_EQUAL and len(diff_text) >= 2 * self.Patch_Margin:
                 # Time for a new patch.
                 if len(patch.diffs) != 0:
                     self.patch_addContext(patch, prepatch_text)
                     patches.append(patch)
                     patch = patch_obj()
                     # Unlike Unidiff, our patch lists have a rolling context.
                     # http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
                     # Update prepatch text & pos to reflect the application of the
                     # just completed patch.
                     prepatch_text = postpatch_text
                     char_count1 = char_count2
 
             # Update the current character count.
             if diff_type != self.DIFF_INSERT:
                 char_count1 += len(diff_text)
             if diff_type != self.DIFF_DELETE:
                 char_count2 += len(diff_text)
 
         # Pick up the leftover patch if not empty.
         if len(patch.diffs) != 0:
             self.patch_addContext(patch, prepatch_text)
             patches.append(patch)
         return patches
 
     def patch_deepCopy(self, patches):
         """Given an array of patches, return another array that is identical.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           Array of Patch objects.
         """
         patchesCopy = []
         for patch in patches:
             patchCopy = patch_obj()
             # No need to deep copy the tuples since they are immutable.
             patchCopy.diffs = patch.diffs[:]
             patchCopy.start1 = patch.start1
             patchCopy.start2 = patch.start2
             patchCopy.length1 = patch.length1
             patchCopy.length2 = patch.length2
             patchesCopy.append(patchCopy)
         return patchesCopy
 
     def patch_apply(self, patches, text):
         """Merge a set of patches onto the text.  Return a patched text, as well
         as a list of true/false values indicating which patches were applied.
 
         Args:
           patches: Array of Patch objects.
           text: Old text.
 
         Returns:
           Two element Array, containing the new text and an array of boolean values.
         """
         if not patches:
             return (text, [])
 
         # Deep copy the patches so that no changes are made to originals.
         patches = self.patch_deepCopy(patches)
 
         nullPadding = self.patch_addPadding(patches)
         text = nullPadding + text + nullPadding
         self.patch_splitMax(patches)
 
         # delta keeps track of the offset between the expected and actual location
         # of the previous patch.  If there are patches expected at positions 10 and
         # 20, but the first patch was found at 12, delta is 2 and the second patch
         # has an effective expected position of 22.
         delta = 0
         results = []
         for patch in patches:
             expected_loc = patch.start2 + delta
             text1 = self.diff_text1(patch.diffs)
             end_loc = -1
             if len(text1) > self.Match_MaxBits:
                 # patch_splitMax will only provide an oversized pattern in the case of
                 # a monster delete.
                 start_loc = self.match_main(
                     text, text1[: self.Match_MaxBits], expected_loc
                 )
                 if start_loc != -1:
                     end_loc = self.match_main(
                         text,
                         text1[-self.Match_MaxBits :],
                         expected_loc + len(text1) - self.Match_MaxBits,
                     )
                     if end_loc == -1 or start_loc >= end_loc:
                         # Can't find valid trailing context.  Drop this patch.
                         start_loc = -1
             else:
                 start_loc = self.match_main(text, text1, expected_loc)
             if start_loc == -1:
                 # No match found.  :(
                 results.append(False)
                 # Subtract the delta for this failed patch from subsequent patches.
                 delta -= patch.length2 - patch.length1
             else:
                 # Found a match.  :)
                 results.append(True)
                 delta = start_loc - expected_loc
                 if end_loc == -1:
                     text2 = text[start_loc : start_loc + len(text1)]
                 else:
                     text2 = text[start_loc : end_loc + self.Match_MaxBits]
                 if text1 == text2:
                     # Perfect match, just shove the replacement text in.
                     text = (
                         text[:start_loc]
                         + self.diff_text2(patch.diffs)
                         + text[start_loc + len(text1) :]
                     )
                 else:
                     # Imperfect match.
                     # Run a diff to get a framework of equivalent indices.
                     diffs = self.diff_main(text1, text2, False)
                     if (
                         len(text1) > self.Match_MaxBits
                         and self.diff_levenshtein(diffs) / float(len(text1))
                         > self.Patch_DeleteThreshold
                     ):
                         # The end points match, but the content is unacceptably bad.
                         results[-1] = False
                     else:
                         self.diff_cleanupSemanticLossless(diffs)
                         index1 = 0
                         for op, data in patch.diffs:
                             if op != self.DIFF_EQUAL:
                                 index2 = self.diff_xIndex(diffs, index1)
                             if op == self.DIFF_INSERT:  # Insertion
                                 text = (
                                     text[: start_loc + index2]
                                     + data
                                     + text[start_loc + index2 :]
                                 )
                             elif op == self.DIFF_DELETE:  # Deletion
                                 text = (
                                     text[: start_loc + index2]
                                     + text[
                                         start_loc
                                         + self.diff_xIndex(diffs, index1 + len(data)) :
                                     ]
                                 )
                             if op != self.DIFF_DELETE:
                                 index1 += len(data)
         # Strip the padding off.
         text = text[len(nullPadding) : -len(nullPadding)]
         return (text, results)
 
     def patch_addPadding(self, patches):
         """Add some padding on text start and end so that edges can match
         something.  Intended to be called only from within patch_apply.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           The padding string added to each side.
         """
         paddingLength = self.Patch_Margin
         nullPadding = ""
         for x in range(1, paddingLength + 1):
             nullPadding += chr(x)
 
         # Bump all the patches forward.
         for patch in patches:
             patch.start1 += paddingLength
             patch.start2 += paddingLength
 
         # Add some padding on start of first diff.
         patch = patches[0]
         diffs = patch.diffs
         if not diffs or diffs[0][0] != self.DIFF_EQUAL:
             # Add nullPadding equality.
             diffs.insert(0, (self.DIFF_EQUAL, nullPadding))
             patch.start1 -= paddingLength  # Should be 0.
             patch.start2 -= paddingLength  # Should be 0.
             patch.length1 += paddingLength
             patch.length2 += paddingLength
         elif paddingLength > len(diffs[0][1]):
             # Grow first equality.
             extraLength = paddingLength - len(diffs[0][1])
             newText = nullPadding[len(diffs[0][1]) :] + diffs[0][1]
             diffs[0] = (diffs[0][0], newText)
             patch.start1 -= extraLength
             patch.start2 -= extraLength
             patch.length1 += extraLength
             patch.length2 += extraLength
 
         # Add some padding on end of last diff.
         patch = patches[-1]
         diffs = patch.diffs
         if not diffs or diffs[-1][0] != self.DIFF_EQUAL:
             # Add nullPadding equality.
             diffs.append((self.DIFF_EQUAL, nullPadding))
             patch.length1 += paddingLength
             patch.length2 += paddingLength
         elif paddingLength > len(diffs[-1][1]):
             # Grow last equality.
             extraLength = paddingLength - len(diffs[-1][1])
             newText = diffs[-1][1] + nullPadding[:extraLength]
             diffs[-1] = (diffs[-1][0], newText)
             patch.length1 += extraLength
             patch.length2 += extraLength
 
         return nullPadding
 
     def patch_splitMax(self, patches):
         """Look through the patches and break up any which are longer than the
         maximum limit of the match algorithm.
         Intended to be called only from within patch_apply.
 
         Args:
           patches: Array of Patch objects.
         """
         patch_size = self.Match_MaxBits
         if patch_size == 0:
             # Python has the option of not splitting strings due to its ability
             # to handle integers of arbitrary precision.
             return
         for x in range(len(patches)):
             if patches[x].length1 <= patch_size:
                 continue
             bigpatch = patches[x]
             # Remove the big old patch.
             del patches[x]
             x -= 1
             start1 = bigpatch.start1
             start2 = bigpatch.start2
             precontext = ""
             while len(bigpatch.diffs) != 0:
                 # Create one of several smaller patches.
                 patch = patch_obj()
                 empty = True
                 patch.start1 = start1 - len(precontext)
                 patch.start2 = start2 - len(precontext)
                 if precontext:
                     patch.length1 = patch.length2 = len(precontext)
                     patch.diffs.append((self.DIFF_EQUAL, precontext))
 
                 while (
                     len(bigpatch.diffs) != 0
                     and patch.length1 < patch_size - self.Patch_Margin
                 ):
                     (diff_type, diff_text) = bigpatch.diffs[0]
                     if diff_type == self.DIFF_INSERT:
                         # Insertions are harmless.
                         patch.length2 += len(diff_text)
                         start2 += len(diff_text)
                         patch.diffs.append(bigpatch.diffs.pop(0))
                         empty = False
                     elif (
                         diff_type == self.DIFF_DELETE
                         and len(patch.diffs) == 1
                         and patch.diffs[0][0] == self.DIFF_EQUAL
                         and len(diff_text) > 2 * patch_size
                     ):
                         # This is a large deletion.  Let it pass in one chunk.
                         patch.length1 += len(diff_text)
                         start1 += len(diff_text)
                         empty = False
                         patch.diffs.append((diff_type, diff_text))
                         del bigpatch.diffs[0]
                     else:
                         # Deletion or equality.  Only take as much as we can stomach.
                         diff_text = diff_text[
                             : patch_size - patch.length1 - self.Patch_Margin
                         ]
                         patch.length1 += len(diff_text)
                         start1 += len(diff_text)
                         if diff_type == self.DIFF_EQUAL:
                             patch.length2 += len(diff_text)
                             start2 += len(diff_text)
                         else:
                             empty = False
 
                         patch.diffs.append((diff_type, diff_text))
                         if diff_text == bigpatch.diffs[0][1]:
                             del bigpatch.diffs[0]
                         else:
                             bigpatch.diffs[0] = (
                                 bigpatch.diffs[0][0],
                                 bigpatch.diffs[0][1][len(diff_text) :],
                             )
 
                 # Compute the head context for the next patch.
                 precontext = self.diff_text2(patch.diffs)
                 precontext = precontext[-self.Patch_Margin :]
                 # Append the end context for this patch.
                 postcontext = self.diff_text1(bigpatch.diffs)[: self.Patch_Margin]
                 if postcontext:
                     patch.length1 += len(postcontext)
                     patch.length2 += len(postcontext)
                     if len(patch.diffs) != 0 and patch.diffs[-1][0] == self.DIFF_EQUAL:
                         patch.diffs[-1] = (
                             self.DIFF_EQUAL,
                             patch.diffs[-1][1] + postcontext,
                         )
                     else:
                         patch.diffs.append((self.DIFF_EQUAL, postcontext))
 
                 if not empty:
                     x += 1
                     patches.insert(x, patch)
 
     def patch_toText(self, patches):
         """Take a list of patches and return a textual representation.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           Text representation of patches.
         """
         text = []
         for patch in patches:
             text.append(str(patch))
         return "".join(text)
 
     def patch_fromText(self, textline):
         """Parse a textual representation of patches and return a list of patch
         objects.
 
         Args:
           textline: Text representation of patches.
 
         Returns:
           Array of Patch objects.
 
         Raises:
           ValueError: If invalid input.
         """
-        if type(textline) == unicode:
+        if type(textline) == str:
             # Patches should be composed of a subset of ascii chars, Unicode not
             # required.  If this encode raises UnicodeEncodeError, patch is invalid.
             textline = textline.encode("ascii")
         patches = []
         if not textline:
             return patches
         text = textline.split("\n")
         while len(text) != 0:
             m = re.match("^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$", text[0])
             if not m:
                 raise ValueError("Invalid patch string: " + text[0])
             patch = patch_obj()
             patches.append(patch)
             patch.start1 = int(m.group(1))
             if m.group(2) == "":
                 patch.start1 -= 1
                 patch.length1 = 1
             elif m.group(2) == "0":
                 patch.length1 = 0
             else:
                 patch.start1 -= 1
                 patch.length1 = int(m.group(2))
 
             patch.start2 = int(m.group(3))
             if m.group(4) == "":
                 patch.start2 -= 1
                 patch.length2 = 1
             elif m.group(4) == "0":
                 patch.length2 = 0
             else:
                 patch.start2 -= 1
                 patch.length2 = int(m.group(4))
 
             del text[0]
 
             while len(text) != 0:
                 if text[0]:
                     sign = text[0][0]
                 else:
                     sign = ""
                 line = urllib.parse.unquote(text[0][1:])
                 line = line.decode("utf-8")
                 if sign == "+":
                     # Insertion.
                     patch.diffs.append((self.DIFF_INSERT, line))
                 elif sign == "-":
                     # Deletion.
                     patch.diffs.append((self.DIFF_DELETE, line))
                 elif sign == " ":
                     # Minor equality.
                     patch.diffs.append((self.DIFF_EQUAL, line))
                 elif sign == "@":
                     # Start of next patch.
                     break
                 elif sign == "":
                     # Blank line?  Whatever.
                     pass
                 else:
                     # WTF?
                     raise ValueError("Invalid patch mode: '%s'\n%s" % (sign, line))
                 del text[0]
         return patches
 
 
 class patch_obj:
     """Class representing one patch operation."""
 
     def __init__(self):
         """Initializes with an empty list of diffs."""
         self.diffs = []
         self.start1 = None
         self.start2 = None
         self.length1 = 0
         self.length2 = 0
 
     def __str__(self):
         """Emmulate GNU diff's format.
         Header: @@ -382,8 +481,9 @@
         Indicies are printed as 1-based, not 0-based.
 
         Returns:
           The GNU diff string.
         """
         if self.length1 == 0:
             coords1 = str(self.start1) + ",0"
         elif self.length1 == 1:
             coords1 = str(self.start1 + 1)
         else:
             coords1 = str(self.start1 + 1) + "," + str(self.length1)
         if self.length2 == 0:
             coords2 = str(self.start2) + ",0"
         elif self.length2 == 1:
             coords2 = str(self.start2 + 1)
         else:
             coords2 = str(self.start2 + 1) + "," + str(self.length2)
         text = ["@@ -", coords1, " +", coords2, " @@\n"]
         # Escape the body of the patch with %xx notation.
         for op, data in self.diffs:
             if op == diff_match_patch.DIFF_INSERT:
                 text.append("+")
             elif op == diff_match_patch.DIFF_DELETE:
                 text.append("-")
             elif op == diff_match_patch.DIFF_EQUAL:
                 text.append(" ")
             # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
             data = data.encode("utf-8")
             text.append(urllib.parse.quote(data, "!~*'();/?:@&=+$,# ") + "\n")
         return "".join(text)

rhodecode/lib/diffs.py

@@ -1,1272 +1,1271 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of diffing helpers, previously part of vcs
 """
 
 import os
 import re
 import bz2
 import gzip
 import time
 
 import collections
 import difflib
 import logging
 import pickle
 from itertools import tee
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
 # define max context, a file with more than this numbers of lines is unusable
 # in browser anyway
 MAX_CONTEXT = 20 * 1024
 DEFAULT_CONTEXT = 3
 
 
 def get_diff_context(request):
     return MAX_CONTEXT if request.GET.get('fullcontext', '') == '1' else DEFAULT_CONTEXT
 
 
 def get_diff_whitespace_flag(request):
     return request.GET.get('ignorews', '') == '1'
 
 
 class OPS(object):
     ADD = 'A'
     MOD = 'M'
     DEL = 'D'
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     # protect against IntOverflow when passing HUGE context
     if context > MAX_CONTEXT:
         context = MAX_CONTEXT
 
-    submodules = filter(lambda o: isinstance(o, SubModuleNode),
-                        [filenode_new, filenode_old])
+    submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError(
                 "Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.commit.repository
     old_commit = filenode_old.commit or repo.EMPTY_COMMIT
     new_commit = filenode_new.commit
 
     vcs_gitdiff = repo.get_diff(
         old_commit, new_commit, filenode_new.path,
         ignore_whitespace, context, path1=filenode_old.path)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __getitem__(self, key):
         return self.diff.__getitem__(key)
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class Action(object):
     """
     Contains constants for the action value of the lines in a parsed diff.
     """
 
     ADD = 'add'
     DELETE = 'del'
     UNMODIFIED = 'unmod'
 
     CONTEXT = 'context'
     OLD_NO_NL = 'old-no-nl'
     NEW_NO_NL = 'new-no-nl'
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
 
     .. note:: Unicode handling
 
        The original diffs are a byte sequence and can contain filenames
        in mixed encodings. This class generally returns `unicode` objects
        since the result is intended for presentation to the user.
 
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
 
     # used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     # collapse ranges of commits over given number
     _collapse_commits_over = 5
 
     def __init__(self, diff, format='gitdiff', diff_limit=None,
                  file_limit=None, show_full_diff=True):
         """
         :param diff: A `Diff` object representing a diff from a vcs backend
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_limit = diff_limit
         self.file_limit = file_limit
         self.show_full_diff = show_full_diff
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
 
         log.debug('Initialized DiffProcessor with %s mode', format)
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._new_parse_gitdiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         """
         self.cur_diff_size += len(string)
 
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return string \
             .replace('&', '&amp;')\
             .replace('<', '&lt;')\
             .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == Action.DELETE:
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == Action.ADD:
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             # only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
 
             diff = map(self._escaper, self.diff_splitter(chunk.diff))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be copy, rename or chmod
 
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     raw_diff, chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
                     exceeds_limit = len(raw_diff) > self.file_limit
 
                     # changed from _escaper function so we validate size of
                     # each file instead of the whole diff
                     # diff will hide big files but still show small ones
                     # from my tests, big files are fairly safe to be parsed
                     # but the browser is the bottleneck
                     if not self.show_full_diff and exceeds_limit:
                         raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(0, [{
                                   'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].items()
                               if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = next(lineiter)
                         if line['action'] not in (
                                 Action.UNMODIFIED, Action.CONTEXT):
                             nextline = next(lineiter)
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _check_large_diff(self):
         if self.diff_limit:
             log.debug('Checking if diff exceeds current diff_limit of %s', self.diff_limit)
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit `%s` Exceeded', self.diff_limit)
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_gitdiff
     def _new_parse_gitdiff(self, inline_diff=True):
         _files = []
 
         # this can be overriden later to a LimitedDiffContainer type
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
             log.debug('parsing diff %r', head)
 
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'old_mode': None,
                 'new_mode': None,
                 'ops': {},
             }
             if head['old_mode']:
                 stats['old_mode'] = head['old_mode']
             if head['new_mode']:
                 stats['new_mode'] = head['new_mode']
             if head['b_mode']:
                 stats['new_mode'] = head['b_mode']
 
             # delete file
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             # new file
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['old_mode'] = None
                 stats['new_mode'] = head['new_file_mode']
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
 
             # modify operation, can be copy, rename or chmod
             else:
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
 
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['renamed'] = (head['rename_from'], head['rename_to'])
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['copied'] = (head['copy_from'], head['copy_to'])
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['new_file'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 # simulate splitlines, so we keep the line end part
                 diff = self.diff_splitter(chunk.diff)
 
                 # append each file to the diff size
                 raw_chunk_size = len(raw_diff)
 
                 exceeds_limit = raw_chunk_size > self.file_limit
                 self.cur_diff_size += raw_chunk_size
 
                 try:
                     # Check each file instead of the whole diff.
                     # Diff will hide big files but still show small ones.
                     # From the tests big files are fairly safe to be parsed
                     # but the browser is the bottleneck.
                     if not self.show_full_diff and exceeds_limit:
                         log.debug('File `%s` exceeds current file_limit of %s',
                                   safe_unicode(head['b_path']), self.file_limit)
                         raise DiffLimitExceeded(
                             'File Limit %s Exceeded', self.file_limit)
 
                     self._check_large_diff()
 
                     raw_diff, chunks, _stats = self._new_parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             # Hide content of deleted node by setting empty chunks
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(
                 0, [{'old_lineno': '',
                      'new_lineno': '',
                      'action': Action.CONTEXT,
                      'line': msg,
                      } for _op, msg in stats['ops'].items()
                     if _op not in [MOD_FILENODE]])
 
             original_filename = safe_unicode(head['a_path'])
             _files.append({
                 'original_filename': original_filename,
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         return diff_container(sorted(_files, key=sorter))
 
     # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
     def _parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     Action.CONTEXT,
                             'line':       line,
                         })
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       Action.CONTEXT,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return ''.join(raw_diff), chunks, stats
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_lines
     def _new_parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 # match header e.g @@ -0,0 +1 @@\n'
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
 
                 lines = []
                 hunk = {
                     'section_header': gr[-1],
                     'source_start': old_line,
                     'source_length': old_end,
                     'target_start': new_line,
                     'target_length': new_end,
                     'lines': lines,
                 }
                 chunks.append(hunk)
 
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                     raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         if affects_old:
                             action = Action.OLD_NO_NL
                         elif affects_new:
                             action = Action.NEW_NO_NL
                         else:
                             raise Exception('invalid context for no newline')
 
                         lines.append({
                             'old_lineno':   None,
                             'new_lineno':   None,
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
 
         return ''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     @classmethod
     def diff_splitter(cls, string):
         """
         Diff split that emulates .splitlines() but works only on \n
         """
         if not string:
             return
         elif string == '\n':
-            yield u'\n'
+            yield '\n'
         else:
 
             has_newline = string.endswith('\n')
             elements = string.split('\n')
             if has_newline:
                 # skip last element as it's empty string from newlines
                 elements = elements[:-1]
 
             len_elements = len(elements)
 
             for cnt, line in enumerate(elements, start=1):
                 last_line = cnt == len_elements
                 if last_line and not has_newline:
                     yield safe_unicode(line)
                 else:
                     yield safe_unicode(line) + '\n'
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering.
 
         :return: A list of dicts with diff information.
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw diff as a byte string
         """
         return self._diff.raw
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         # TODO(marcink): not sure how to pass in translator
         # here in an efficient way, leave the _ for proper gettext extraction
         _ = lambda s: s
 
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s" class="tooltip"
                 title="%(title)s">%(label)s</a>''' % {
                     'title': _('Click to select line'),
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
 
                     if change['action'] != Action.CONTEXT:
                         anchor_link = True
                     else:
                         anchor_link = False
 
                     ###########################################################
                     # COMMENT ICONS
                     ###########################################################
                     _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
 
                     if enable_comments and change['action'] != Action.CONTEXT:
                         _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
 
                     _html.append('''</span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>\n''')
 
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     code_classes = [code_class]
                     if (not enable_comments or
                             change['action'] == Action.CONTEXT):
                         code_classes.append('no-comment')
                     _html.append('\t<td class="%s">' % ' '.join(code_classes))
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
     def get_context_of_line(
             self, path, diff_line=None, context_before=3, context_after=3):
         """
         Returns the context lines for the specified diff line.
 
         :type diff_line: :class:`DiffLineNumber`
         """
         assert self.parsed, "DiffProcessor is not initialized."
 
         if None not in diff_line:
             raise ValueError(
                 "Cannot specify both line numbers: {}".format(diff_line))
 
         file_diff = self._get_file_diff(path)
         chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
 
         first_line_to_include = max(idx - context_before, 0)
         first_line_after_context = idx + context_after + 1
         context_lines = chunk[first_line_to_include:first_line_after_context]
 
         line_contents = [
             _context_line(line) for line in context_lines
             if _is_diff_content(line)]
         # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
         # Once they are fixed, we can drop this line here.
         if line_contents:
             line_contents[-1] = (
                 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
         return line_contents
 
     def find_context(self, path, context, offset=0):
         """
         Finds the given `context` inside of the diff.
 
         Use the parameter `offset` to specify which offset the target line has
         inside of the given `context`. This way the correct diff line will be
         returned.
 
         :param offset: Shall be used to specify the offset of the main line
             within the given `context`.
         """
         if offset < 0 or offset >= len(context):
             raise ValueError(
                 "Only positive values up to the length of the context "
                 "minus one are allowed.")
 
         matches = []
         file_diff = self._get_file_diff(path)
 
         for chunk in file_diff['chunks']:
             context_iter = iter(context)
             for line_idx, line in enumerate(chunk):
                 try:
                     if _context_line(line) == next(context_iter):
                         continue
                 except StopIteration:
                     matches.append((line_idx, chunk))
                 context_iter = iter(context)
 
         # Increment position and triger StopIteration
         # if we had a match at the end
         line_idx += 1
         try:
             next(context_iter)
         except StopIteration:
             matches.append((line_idx, chunk))
 
         effective_offset = len(context) - offset
         found_at_diff_lines = [
             _line_to_diff_line_number(chunk[idx - effective_offset])
             for idx, chunk in matches]
 
         return found_at_diff_lines
 
     def _get_file_diff(self, path):
         for file_diff in self.parsed_diff:
             if file_diff['filename'] == path:
                 break
         else:
             raise FileNotInDiffException("File {} not in diff".format(path))
         return file_diff
 
     def _find_chunk_line_index(self, file_diff, diff_line):
         for chunk in file_diff['chunks']:
             for idx, line in enumerate(chunk):
                 if line['old_lineno'] == diff_line.old:
                     return chunk, idx
                 if line['new_lineno'] == diff_line.new:
                     return chunk, idx
         raise LineNotInDiffException(
             "The line {} is not part of the diff.".format(diff_line))
 
 
 def _is_diff_content(line):
     return line['action'] in (
         Action.UNMODIFIED, Action.ADD, Action.DELETE)
 
 
 def _context_line(line):
     return (line['action'], line['line'])
 
 
 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
 
 
 def _line_to_diff_line_number(line):
     new_line_no = line['new_lineno'] or None
     old_line_no = line['old_lineno'] or None
     return DiffLineNumber(old=old_line_no, new=new_line_no)
 
 
 class FileNotInDiffException(Exception):
     """
     Raised when the context for a missing file is requested.
 
     If you request the context for a line in a file which is not part of the
     given diff, then this exception is raised.
     """
 
 
 class LineNotInDiffException(Exception):
     """
     Raised when the context for a missing line is requested.
 
     If you request the context for a line in a file and this line is not
     part of the given diff, then this exception is raised.
     """
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 # NOTE(marcink): if diffs.mako change, probably this
 # needs a bump to next version
 CURRENT_DIFF_VERSION = 'v5'
 
 
 def _cleanup_cache_file(cached_diff_file):
     # cleanup file to not store it "damaged"
     try:
         os.remove(cached_diff_file)
     except Exception:
         log.exception('Failed to cleanup path %s', cached_diff_file)
 
 
 def _get_compression_mode(cached_diff_file):
     mode = 'bz2'
     if 'mode:plain' in cached_diff_file:
         mode = 'plain'
     elif 'mode:gzip' in cached_diff_file:
         mode = 'gzip'
     return mode
 
 
 def cache_diff(cached_diff_file, diff, commits):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': diff,
         'commits': commits
     }
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         else:
             with bz2.BZ2File(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
     except Exception:
         log.warn('Failed to save cache', exc_info=True)
         _cleanup_cache_file(cached_diff_file)
 
     log.debug('Saved diff cache under %s in %.4fs', cached_diff_file, time.time() - start)
 
 
 def load_cached_diff(cached_diff_file):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     default_struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': None,
         'commits': None
     }
 
     has_cache = os.path.isfile(cached_diff_file)
     if not has_cache:
         log.debug('Reading diff cache file failed %s', cached_diff_file)
         return default_struct
 
     data = None
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         else:
             with bz2.BZ2File(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
     except Exception:
         log.warn('Failed to read diff cache file', exc_info=True)
 
     if not data:
         data = default_struct
 
     if not isinstance(data, dict):
         # old version of data ?
         data = default_struct
 
     # check version
     if data.get('version') != CURRENT_DIFF_VERSION:
         # purge cache
         _cleanup_cache_file(cached_diff_file)
         return default_struct
 
     log.debug('Loaded diff cache from %s in %.4fs', cached_diff_file, time.time() - start)
 
     return data
 
 
 def generate_diff_cache_key(*args):
     """
     Helper to generate a cache key using arguments
     """
     def arg_mapper(input_param):
         input_param = safe_str(input_param)
         # we cannot allow '/' in arguments since it would allow
         # subdirectory usage
         input_param.replace('/', '_')
         return input_param or None  # prevent empty string arguments
 
     return '_'.join([
         '{}' for i in range(len(args))]).format(*map(arg_mapper, args))
 
 
 def diff_cache_exist(cache_storage, *args):
     """
     Based on all generated arguments check and return a cache path
     """
     args = list(args) + ['mode:gzip']
     cache_key = generate_diff_cache_key(*args)
     cache_file_path = os.path.join(cache_storage, cache_key)
     # prevent path traversal attacks using some param that have e.g '../../'
     if not os.path.abspath(cache_file_path).startswith(cache_storage):
         raise ValueError('Final path must be within {}'.format(cache_storage))
 
     return cache_file_path

rhodecode/lib/feedgenerator/feedgenerator.py

@@ -1,444 +1,444 b''
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
 #     1. Redistributions of source code must retain the above copyright notice,
 #        this list of conditions and the following disclaimer.
 #
 #     2. Redistributions in binary form must reproduce the above copyright
 #        notice, this list of conditions and the following disclaimer in the
 #        documentation and/or other materials provided with the distribution.
 #
 #     3. Neither the name of Django nor the names of its contributors may be used
 #        to endorse or promote products derived from this software without
 #        specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """
 For definitions of the different versions of RSS, see:
 http://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004/02/04/incompatible-rss
 """
 
 
 import datetime
-from io import StringIO
+import io
 
 import pytz
 from six.moves.urllib import parse as urlparse
 
 from rhodecode.lib.feedgenerator import datetime_safe
 from rhodecode.lib.feedgenerator.utils import SimplerXMLGenerator, iri_to_uri, force_text
 
 
 #### The following code comes from ``django.utils.feedgenerator`` ####
 
 
 def rfc2822_date(date):
     # We can't use strftime() because it produces locale-dependent results, so
     # we have to map english month and day names manually
     months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',)
     days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     # We do this ourselves to be timezone aware, email.Utils is not tz aware.
     dow = days[date.weekday()]
     month = months[date.month - 1]
     time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month))
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + '-0000'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d%02d' % (hour, minute)
 
 
 def rfc3339_date(date):
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     time_str = date.strftime('%Y-%m-%dT%H:%M:%S')
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + 'Z'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d:%02d' % (hour, minute)
 
 
 def get_tag_uri(url, date):
     """
     Creates a TagURI.
 
     See http://web.archive.org/web/20110514113830/http://diveintomark.org/archives/2004/05/28/howto-atom-id
     """
     bits = urlparse(url)
     d = ''
     if date is not None:
         d = ',%s' % datetime_safe.new_datetime(date).strftime('%Y-%m-%d')
     return 'tag:%s%s:%s/%s' % (bits.hostname, d, bits.path, bits.fragment)
 
 
 class SyndicationFeed(object):
     """Base class for all syndication feeds. Subclasses should provide write()"""
 
     def __init__(self, title, link, description, language=None, author_email=None,
                  author_name=None, author_link=None, subtitle=None, categories=None,
                  feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs):
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [force_text(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         self.feed = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'language': to_unicode(language),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'subtitle': to_unicode(subtitle),
             'categories': categories or (),
             'feed_url': iri_to_uri(feed_url),
             'feed_copyright': to_unicode(feed_copyright),
             'id': feed_guid or link,
             'ttl': ttl,
         }
         self.feed.update(kwargs)
         self.items = []
 
     def add_item(self, title, link, description, author_email=None,
                  author_name=None, author_link=None, pubdate=None, comments=None,
                  unique_id=None, unique_id_is_permalink=None, enclosure=None,
                  categories=(), item_copyright=None, ttl=None, updateddate=None,
                  enclosures=None, **kwargs):
         """
         Adds an item to the feed. All args are expected to be Python Unicode
         objects except pubdate and updateddate, which are datetime.datetime
         objects, and enclosures, which is an iterable of instances of the
         Enclosure class.
         """
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [to_unicode(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         if enclosure is None:
             enclosures = [] if enclosures is None else enclosures
 
         item = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'pubdate': pubdate,
             'updateddate': updateddate,
             'comments': to_unicode(comments),
             'unique_id': to_unicode(unique_id),
             'unique_id_is_permalink': unique_id_is_permalink,
             'enclosures': enclosures,
             'categories': categories or (),
             'item_copyright': to_unicode(item_copyright),
             'ttl': ttl,
         }
         item.update(kwargs)
         self.items.append(item)
 
     def num_items(self):
         return len(self.items)
 
     def root_attributes(self):
         """
         Return extra attributes to place on the root (i.e. feed/channel) element.
         Called from write().
         """
         return {}
 
     def add_root_elements(self, handler):
         """
         Add elements in the root (i.e. feed/channel) element. Called
         from write().
         """
         pass
 
     def item_attributes(self, item):
         """
         Return extra attributes to place on each item (i.e. item/entry) element.
         """
         return {}
 
     def add_item_elements(self, handler, item):
         """
         Add elements on each item (i.e. item/entry) element.
         """
         pass
 
     def write(self, outfile, encoding):
         """
         Outputs the feed in the given encoding to outfile, which is a file-like
         object. Subclasses should override this.
         """
         raise NotImplementedError('subclasses of SyndicationFeed must provide a write() method')
 
     def writeString(self, encoding):
         """
         Returns the feed in the given encoding as a string.
         """
-        s = StringIO()
+        s = io.StringIO()
         self.write(s, encoding)
         return s.getvalue()
 
     def latest_post_date(self):
         """
         Returns the latest item's pubdate or updateddate. If no items
         have either of these attributes this returns the current UTC date/time.
         """
         latest_date = None
         date_keys = ('updateddate', 'pubdate')
 
         for item in self.items:
             for date_key in date_keys:
                 item_date = item.get(date_key)
                 if item_date:
                     if latest_date is None or item_date > latest_date:
                         latest_date = item_date
 
         # datetime.now(tz=utc) is slower, as documented in django.utils.timezone.now
         return latest_date or datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
 
 
 class Enclosure(object):
     """Represents an RSS enclosure"""
     def __init__(self, url, length, mime_type):
         """All args are expected to be Python Unicode objects"""
         self.length, self.mime_type = length, mime_type
         self.url = iri_to_uri(url)
 
 
 class RssFeed(SyndicationFeed):
     content_type = 'application/rss+xml; charset=utf-8'
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement("rss", self.rss_attributes())
         handler.startElement("channel", self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         self.endChannelElement(handler)
         handler.endElement("rss")
 
     def rss_attributes(self):
         return {"version": self._version,
                 "xmlns:atom": "http://www.w3.org/2005/Atom"}
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement('item', self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("item")
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", self.feed['link'])
         handler.addQuickElement("description", self.feed['description'])
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("atom:link", None, {"rel": "self", "href": self.feed['feed_url']})
         if self.feed['language'] is not None:
             handler.addQuickElement("language", self.feed['language'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", cat)
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("copyright", self.feed['feed_copyright'])
         handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()))
         if self.feed['ttl'] is not None:
             handler.addQuickElement("ttl", self.feed['ttl'])
 
     def endChannelElement(self, handler):
         handler.endElement("channel")
 
 
 class RssUserland091Feed(RssFeed):
     _version = "0.91"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
 
 class Rss201rev2Feed(RssFeed):
     # Spec: http://blogs.law.harvard.edu/tech/rss
     _version = "2.0"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
         # Author information.
         if item["author_name"] and item["author_email"]:
             handler.addQuickElement("author", "%s (%s)" % (item['author_email'], item['author_name']))
         elif item["author_email"]:
             handler.addQuickElement("author", item["author_email"])
         elif item["author_name"]:
             handler.addQuickElement(
                 "dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}
             )
 
         if item['pubdate'] is not None:
             handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']))
         if item['comments'] is not None:
             handler.addQuickElement("comments", item['comments'])
         if item['unique_id'] is not None:
             guid_attrs = {}
             if isinstance(item.get('unique_id_is_permalink'), bool):
                 guid_attrs['isPermaLink'] = str(item['unique_id_is_permalink']).lower()
             handler.addQuickElement("guid", item['unique_id'], guid_attrs)
         if item['ttl'] is not None:
             handler.addQuickElement("ttl", item['ttl'])
 
         # Enclosure.
         if item['enclosures']:
             enclosures = list(item['enclosures'])
             if len(enclosures) > 1:
                 raise ValueError(
                     "RSS feed items may only have one enclosure, see "
                     "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
                 )
             enclosure = enclosures[0]
             handler.addQuickElement('enclosure', '', {
                 'url': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", cat)
 
 
 class Atom1Feed(SyndicationFeed):
     # Spec: https://tools.ietf.org/html/rfc4287
     content_type = 'application/atom+xml; charset=utf-8'
     ns = "http://www.w3.org/2005/Atom"
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement('feed', self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         handler.endElement("feed")
 
     def root_attributes(self):
         if self.feed['language'] is not None:
             return {"xmlns": self.ns, "xml:lang": self.feed['language']}
         else:
             return {"xmlns": self.ns}
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", "", {"rel": "alternate", "href": self.feed['link']})
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']})
         handler.addQuickElement("id", self.feed['id'])
         handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()))
         if self.feed['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", self.feed['author_name'])
             if self.feed['author_email'] is not None:
                 handler.addQuickElement("email", self.feed['author_email'])
             if self.feed['author_link'] is not None:
                 handler.addQuickElement("uri", self.feed['author_link'])
             handler.endElement("author")
         if self.feed['subtitle'] is not None:
             handler.addQuickElement("subtitle", self.feed['subtitle'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", "", {"term": cat})
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("rights", self.feed['feed_copyright'])
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement("entry", self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("entry")
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"})
 
         if item['pubdate'] is not None:
             handler.addQuickElement('published', rfc3339_date(item['pubdate']))
 
         if item['updateddate'] is not None:
             handler.addQuickElement('updated', rfc3339_date(item['updateddate']))
 
         # Author information.
         if item['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", item['author_name'])
             if item['author_email'] is not None:
                 handler.addQuickElement("email", item['author_email'])
             if item['author_link'] is not None:
                 handler.addQuickElement("uri", item['author_link'])
             handler.endElement("author")
 
         # Unique ID.
         if item['unique_id'] is not None:
             unique_id = item['unique_id']
         else:
             unique_id = get_tag_uri(item['link'], item['pubdate'])
         handler.addQuickElement("id", unique_id)
 
         # Summary.
         if item['description'] is not None:
             handler.addQuickElement("summary", item['description'], {"type": "html"})
 
         # Enclosures.
         for enclosure in item['enclosures']:
             handler.addQuickElement('link', '', {
                 'rel': 'enclosure',
                 'href': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", "", {"term": cat})
 
         # Rights.
         if item['item_copyright'] is not None:
             handler.addQuickElement("rights", item['item_copyright'])
 
 
 # This isolates the decision of what the system default is, so calling code can
 # do "feedgenerator.DefaultFeed" instead of "feedgenerator.Rss201rev2Feed".
 DefaultFeed = Rss201rev2Feed
  No newline at end of file

rhodecode/lib/hooks_base.py

@@ -1,538 +1,538 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
-    vcs_part = safe_str(u'.%s' % repo.repo_type)
+    vcs_part = safe_str('.%s' % repo.repo_type)
     size_vcs, size_root, size_total = _get_scm_size(vcs_part,
                                                     repo.repo_full_path)
     msg = ('Repository `%s` size summary %s:%s repo:%s total:%s\n'
            % (repo.repo_name, vcs_part, size_vcs, size_root, size_total))
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
         if extras.commit_ids and extras.check_branch_perms:
 
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = 'Branch `{}` changes rejected by rule {}. ' \
                                    'FORCE PUSH FORBIDDEN.'.format(branch_name, rule)
                 else:
                     halt_message = 'Branch `{}` changes rejected by rule {}.'.format(
                         branch_name, rule)
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_pull_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `%s`' % (extras.repository,)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_push_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
 
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = 'Released lock on repo `{}`\n'.format(safe_str(extras.repository))
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='branch', ref_name=safe_str(branch_name)))
 
         for book_name in extras.new_refs['bookmarks']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name)))
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = (
         'Repository `%s` locked by user `%s`. Reason:`%s`'
         % (repo_name, user_name, reason))
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/logging_formatter.py

@@ -1,187 +1,187 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import sys
 import logging
 
 
-BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
 
 # Sequences
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[0;%dm"
 BOLD_SEQ = "\033[1m"
 
 COLORS = {
     'CRITICAL': MAGENTA,
     'ERROR': RED,
     'WARNING': CYAN,
     'INFO': GREEN,
     'DEBUG': BLUE,
     'SQL': YELLOW
 }
 
 
 def _inject_req_id(record, with_prefix=True):
     from pyramid.threadlocal import get_current_request
     dummy = '00000000-0000-0000-0000-000000000000'
     req_id = None
 
     req = get_current_request()
     if req:
         req_id = getattr(req, 'req_id', None)
     if with_prefix:
         req_id = 'req_id:%-36s' % (req_id or dummy)
     else:
         req_id = (req_id or dummy)
     record.req_id = req_id
 
 
 def _add_log_to_debug_bucket(formatted_record):
     from pyramid.threadlocal import get_current_request
     req = get_current_request()
     if req:
         req.req_id_bucket.append(formatted_record)
 
 
 def one_space_trim(s):
     if s.find("  ") == -1:
         return s
     else:
         s = s.replace('  ', ' ')
         return one_space_trim(s)
 
 
 def format_sql(sql):
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
         .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
         .replace('FROM', '\n\tFROM')\
         .replace('ORDER BY', '\n\tORDER BY')\
         .replace('LIMIT', '\n\tLIMIT')\
         .replace('WHERE', '\n\tWHERE')\
         .replace('AND', '\n\tAND')\
         .replace('LEFT', '\n\tLEFT')\
         .replace('INNER', '\n\tINNER')\
         .replace('INSERT', '\n\tINSERT')\
         .replace('DELETE', '\n\tDELETE')
     return sql
 
 
 class ExceptionAwareFormatter(logging.Formatter):
     """
     Extended logging formatter which prints out remote tracebacks.
     """
 
     def formatException(self, ei):
         ex_type, ex_value, ex_tb = ei
 
         local_tb = logging.Formatter.formatException(self, ei)
         if hasattr(ex_value, '_vcs_server_traceback'):
 
             def formatRemoteTraceback(remote_tb_lines):
                 result = ["\n +--- This exception occured remotely on VCSServer - Remote traceback:\n\n"]
                 result.append(remote_tb_lines)
                 result.append("\n +--- End of remote traceback\n")
                 return result
 
             try:
                 if ex_type is not None and ex_value is None and ex_tb is None:
                     # possible old (3.x) call syntax where caller is only
                     # providing exception object
                     if type(ex_type) is not type:
                         raise TypeError(
                             "invalid argument: ex_type should be an exception "
                             "type, or just supply no arguments at all")
                 if ex_type is None and ex_tb is None:
                     ex_type, ex_value, ex_tb = sys.exc_info()
 
                 remote_tb = getattr(ex_value, "_vcs_server_traceback", None)
 
                 if remote_tb:
                     remote_tb = formatRemoteTraceback(remote_tb)
                     return local_tb + ''.join(remote_tb)
             finally:
                 # clean up cycle to traceback, to allow proper GC
                 del ex_type, ex_value, ex_tb
 
         return local_tb
 
 
 class RequestTrackingFormatter(ExceptionAwareFormatter):
     def format(self, record):
         _inject_req_id(record)
         def_record = logging.Formatter.format(self, record)
         _add_log_to_debug_bucket(def_record)
         return def_record
 
 
 class ColorFormatter(ExceptionAwareFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorRequestTrackingFormatter(RequestTrackingFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorRequestTrackingFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorFormatterSql(logging.Formatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
 
         start = COLOR_SEQ % (COLORS['SQL'])
         def_record = format_sql(logging.Formatter.format(self, record))
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 # marcink: needs to stay with this name for backward .ini compatability
 Pyro4AwareFormatter = ExceptionAwareFormatter

rhodecode/lib/markup_renderer.py

@@ -1,580 +1,580 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Renderer for markup languages with ability to parse using rst or markdown
 """
 
 import re
 import os
 import lxml
 import logging
 import urllib.parse
 import bleach
 
 from mako.lookup import TemplateLookup
 from mako.template import Template as MakoTemplate
 
 from docutils.core import publish_parts
 from docutils.parsers.rst import directives
 from docutils import writers
 from docutils.writers import html4css1
 import markdown
 
 from rhodecode.lib.markdown_ext import GithubFlavoredMarkdownExtension
 from rhodecode.lib.utils2 import (safe_unicode, md5_safe, MENTIONS_REGEX)
 
 log = logging.getLogger(__name__)
 
 # default renderer used to generate automated comments
 DEFAULT_COMMENTS_RENDERER = 'rst'
 
 try:
     from lxml.html import fromstring
     from lxml.html import tostring
 except ImportError:
     log.exception('Failed to import lxml')
     fromstring = None
     tostring = None
 
 
 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
     """
     Custom HTML Translator used for sandboxing potential
     JS injections in ref links
     """
     def visit_literal_block(self, node):
         self.body.append(self.starttag(node, 'pre', CLASS='codehilite literal-block'))
 
     def visit_reference(self, node):
         if 'refuri' in node.attributes:
             refuri = node['refuri']
             if ':' in refuri:
                 prefix, link = refuri.lstrip().split(':', 1)
                 prefix = prefix or ''
 
                 if prefix.lower() == 'javascript':
                     # we don't allow javascript type of refs...
                     node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
 
         # old style class requires this...
         return html4css1.HTMLTranslator.visit_reference(self, node)
 
 
 class RhodeCodeWriter(writers.html4css1.Writer):
     def __init__(self):
         writers.Writer.__init__(self)
         self.translator_class = CustomHTMLTranslator
 
 
 def relative_links(html_source, server_paths):
     if not html_source:
         return html_source
 
     if not fromstring and tostring:
         return html_source
 
     try:
         doc = lxml.html.fromstring(html_source)
     except Exception:
         return html_source
 
     for el in doc.cssselect('img, video'):
         src = el.attrib.get('src')
         if src:
             el.attrib['src'] = relative_path(src, server_paths['raw'])
 
     for el in doc.cssselect('a:not(.gfm)'):
         src = el.attrib.get('href')
         if src:
             raw_mode = el.attrib['href'].endswith('?raw=1')
             if raw_mode:
                 el.attrib['href'] = relative_path(src, server_paths['raw'])
             else:
                 el.attrib['href'] = relative_path(src, server_paths['standard'])
 
     return lxml.html.tostring(doc)
 
 
 def relative_path(path, request_path, is_repo_file=None):
     """
     relative link support, path is a rel path, and request_path is current
     server path (not absolute)
 
     e.g.
 
     path = '../logo.png'
     request_path= '/repo/files/path/file.md'
     produces: '/repo/files/logo.png'
     """
     # TODO(marcink): unicode/str support ?
     # maybe=> safe_unicode(urllib.quote(safe_str(final_path), '/:'))
 
     def dummy_check(p):
         return True  # assume default is a valid file path
 
     is_repo_file = is_repo_file or dummy_check
     if not path:
         return request_path
 
     path = safe_unicode(path)
     request_path = safe_unicode(request_path)
 
-    if path.startswith((u'data:', u'javascript:', u'#', u':')):
+    if path.startswith(('data:', 'javascript:', '#', ':')):
         # skip data, anchor, invalid links
         return path
 
     is_absolute = bool(urllib.parse.urlparse(path).netloc)
     if is_absolute:
         return path
 
     if not request_path:
         return path
 
-    if path.startswith(u'/'):
+    if path.startswith('/'):
         path = path[1:]
 
-    if path.startswith(u'./'):
+    if path.startswith('./'):
         path = path[2:]
 
     parts = request_path.split('/')
     # compute how deep we need to traverse the request_path
     depth = 0
 
     if is_repo_file(request_path):
         # if request path is a VALID file, we use a relative path with
         # one level up
         depth += 1
 
-    while path.startswith(u'../'):
+    while path.startswith('../'):
         depth += 1
         path = path[3:]
 
     if depth > 0:
         parts = parts[:-depth]
 
     parts.append(path)
-    final_path = u'/'.join(parts).lstrip(u'/')
+    final_path = '/'.join(parts).lstrip('/')
 
-    return u'/' + final_path
+    return '/' + final_path
 
 
 _cached_markdown_renderer = None
 
 
 def get_markdown_renderer(extensions, output_format):
     global _cached_markdown_renderer
 
     if _cached_markdown_renderer is None:
         _cached_markdown_renderer = markdown.Markdown(
             extensions=extensions,
             enable_attributes=False, output_format=output_format)
     return _cached_markdown_renderer
 
 
 _cached_markdown_renderer_flavored = None
 
 
 def get_markdown_renderer_flavored(extensions, output_format):
     global _cached_markdown_renderer_flavored
 
     if _cached_markdown_renderer_flavored is None:
         _cached_markdown_renderer_flavored = markdown.Markdown(
             extensions=extensions + [GithubFlavoredMarkdownExtension()],
             enable_attributes=False, output_format=output_format)
     return _cached_markdown_renderer_flavored
 
 
 class MarkupRenderer(object):
     RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
 
     MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
     RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
     JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
     PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
 
     URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                          r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
 
     MENTION_PAT = re.compile(MENTIONS_REGEX)
 
     extensions = ['markdown.extensions.codehilite', 'markdown.extensions.extra',
                   'markdown.extensions.def_list', 'markdown.extensions.sane_lists']
 
     output_format = 'html4'
 
     # extension together with weights. Lower is first means we control how
     # extensions are attached to readme names with those.
     PLAIN_EXTS = [
         # prefer no extension
         ('', 0),  # special case that renders READMES names without extension
         ('.text', 2), ('.TEXT', 2),
         ('.txt', 3), ('.TXT', 3)
     ]
 
     RST_EXTS = [
         ('.rst', 1), ('.rest', 1),
         ('.RST', 2), ('.REST', 2)
     ]
 
     MARKDOWN_EXTS = [
         ('.md', 1), ('.MD', 1),
         ('.mkdn', 2), ('.MKDN', 2),
         ('.mdown', 3), ('.MDOWN', 3),
         ('.markdown', 4), ('.MARKDOWN', 4)
     ]
 
     def _detect_renderer(self, source, filename=None):
         """
         runs detection of what renderer should be used for generating html
         from a markup language
 
         filename can be also explicitly a renderer name
 
         :param source:
         :param filename:
         """
 
         if MarkupRenderer.MARKDOWN_PAT.findall(filename):
             detected_renderer = 'markdown'
         elif MarkupRenderer.RST_PAT.findall(filename):
             detected_renderer = 'rst'
         elif MarkupRenderer.JUPYTER_PAT.findall(filename):
             detected_renderer = 'jupyter'
         elif MarkupRenderer.PLAIN_PAT.findall(filename):
             detected_renderer = 'plain'
         else:
             detected_renderer = 'plain'
 
         return getattr(MarkupRenderer, detected_renderer)
 
     @classmethod
     def bleach_clean(cls, text):
         from .bleach_whitelist import markdown_attrs, markdown_tags
         allowed_tags = markdown_tags
         allowed_attrs = markdown_attrs
 
         try:
             return bleach.clean(text, tags=allowed_tags, attributes=allowed_attrs)
         except Exception:
             return 'UNPARSEABLE TEXT'
 
     @classmethod
     def renderer_from_filename(cls, filename, exclude):
         """
         Detect renderer markdown/rst from filename and optionally use exclude
         list to remove some options. This is mostly used in helpers.
         Returns None when no renderer can be detected.
         """
         def _filter(elements):
             if isinstance(exclude, (list, tuple)):
                 return [x for x in elements if x not in exclude]
             return elements
 
         if filename.endswith(
                 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
             return 'markdown'
         if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
             return 'rst'
 
         return None
 
     def render(self, source, filename=None):
         """
         Renders a given filename using detected renderer
         it detects renderers based on file extension or mimetype.
         At last it will just do a simple html replacing new lines with <br/>
 
         :param file_name:
         :param source:
         """
 
         renderer = self._detect_renderer(source, filename)
         readme_data = renderer(source)
         return readme_data
 
     @classmethod
     def _flavored_markdown(cls, text):
         """
         Github style flavored markdown
 
         :param text:
         """
 
         # Extract pre blocks.
         extractions = {}
 
         def pre_extraction_callback(matchobj):
             digest = md5_safe(matchobj.group(0))
             extractions[digest] = matchobj.group(0)
             return "{gfm-extraction-%s}" % digest
         pattern = re.compile(r'<pre>.*?</pre>', re.MULTILINE | re.DOTALL)
         text = re.sub(pattern, pre_extraction_callback, text)
 
         # Prevent foo_bar_baz from ending up with an italic word in the middle.
         def italic_callback(matchobj):
             s = matchobj.group(0)
             if list(s).count('_') >= 2:
                 return s.replace('_', r'\_')
             return s
         text = re.sub(r'^(?! {4}|\t)\w+_\w+_\w[\w_]*', italic_callback, text)
 
         # Insert pre block extractions.
         def pre_insert_callback(matchobj):
             return '\n\n' + extractions[matchobj.group(1)]
         text = re.sub(r'\{gfm-extraction-([0-9a-f]{32})\}',
                       pre_insert_callback, text)
 
         return text
 
     @classmethod
     def urlify_text(cls, text):
         def url_func(match_obj):
             url_full = match_obj.groups()[0]
             return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
 
         return cls.URL_PAT.sub(url_func, text)
 
     @classmethod
     def convert_mentions(cls, text, mode):
         mention_pat = cls.MENTION_PAT
 
         def wrapp(match_obj):
             uname = match_obj.groups()[0]
             hovercard_url = "pyroutes.url('hovercard_username', {'username': '%s'});" % uname
 
             if mode == 'markdown':
                 tmpl = '<strong class="tooltip-hovercard" data-hovercard-alt="{uname}" data-hovercard-url="{hovercard_url}">@{uname}</strong>'
             elif mode == 'rst':
                 tmpl = ' **@{uname}** '
             else:
                 raise ValueError('mode must be rst or markdown')
 
             return tmpl.format(**{'uname': uname,
                                   'hovercard_url': hovercard_url})
 
         return mention_pat.sub(wrapp, text).strip()
 
     @classmethod
     def plain(cls, source, universal_newline=True, leading_newline=True):
         source = safe_unicode(source)
         if universal_newline:
             newline = '\n'
             source = newline.join(source.splitlines())
 
         rendered_source = cls.urlify_text(source)
         source = ''
         if leading_newline:
             source += '<br />'
         source += rendered_source.replace("\n", '<br />')
 
         rendered = cls.bleach_clean(source)
         return rendered
 
     @classmethod
     def markdown(cls, source, safe=True, flavored=True, mentions=False,
                  clean_html=True):
         """
         returns markdown rendered code cleaned by the bleach library
         """
 
         if flavored:
             markdown_renderer = get_markdown_renderer_flavored(
                 cls.extensions, cls.output_format)
         else:
             markdown_renderer = get_markdown_renderer(
                 cls.extensions, cls.output_format)
 
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='markdown')
             # we extracted mentions render with this using Mentions false
             return cls.markdown(mention_hl, safe=safe, flavored=flavored,
                                 mentions=False)
 
         source = safe_unicode(source)
 
         try:
             if flavored:
                 source = cls._flavored_markdown(source)
             rendered = markdown_renderer.convert(source)
         except Exception:
             log.exception('Error when rendering Markdown')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 rendered = cls.plain(source)
             else:
                 raise
 
         if clean_html:
             rendered = cls.bleach_clean(rendered)
         return rendered
 
     @classmethod
     def rst(cls, source, safe=True, mentions=False, clean_html=False):
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='rst')
             # we extracted mentions render with this using Mentions false
             return cls.rst(mention_hl, safe=safe, mentions=False)
 
         source = safe_unicode(source)
         try:
             docutils_settings = dict(
                 [(alias, None) for alias in
                  cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
 
             docutils_settings.update({
                 'input_encoding': 'unicode',
                 'report_level': 4,
                 'syntax_highlight': 'short',
             })
 
             for k, v in docutils_settings.items():
                 directives.register_directive(k, v)
 
             parts = publish_parts(source=source,
                                   writer=RhodeCodeWriter(),
                                   settings_overrides=docutils_settings)
             rendered = parts["fragment"]
             if clean_html:
                 rendered = cls.bleach_clean(rendered)
             return parts['html_title'] + rendered
         except Exception:
             log.exception('Error when rendering RST')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 return cls.plain(source)
             else:
                 raise
 
     @classmethod
     def jupyter(cls, source, safe=True):
         from rhodecode.lib import helpers
 
         from traitlets.config import Config
         import nbformat
         from nbconvert import HTMLExporter
         from nbconvert.preprocessors import Preprocessor
 
         class CustomHTMLExporter(HTMLExporter):
             def _template_file_default(self):
                 return 'basic'
 
         class Sandbox(Preprocessor):
 
             def preprocess(self, nb, resources):
                 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
                 for cell in nb['cells']:
                     if not safe:
                         continue
 
                     if 'outputs' in cell:
                         for cell_output in cell['outputs']:
                             if 'data' in cell_output:
                                 if 'application/javascript' in cell_output['data']:
                                     cell_output['data']['text/plain'] = sandbox_text
                                     cell_output['data'].pop('application/javascript', None)
 
                     if 'source' in cell and cell['cell_type'] == 'markdown':
                         # sanitize similar like in markdown
                         cell['source'] = cls.bleach_clean(cell['source'])
 
                 return nb, resources
 
         def _sanitize_resources(input_resources):
             """
             Skip/sanitize some of the CSS generated and included in jupyter
             so it doesn't messes up UI so much
             """
 
             # TODO(marcink): probably we should replace this with whole custom
             # CSS set that doesn't screw up, but jupyter generated html has some
             # special markers, so it requires Custom HTML exporter template with
             # _default_template_path_default, to achieve that
 
             # strip the reset CSS
             input_resources[0] = input_resources[0][input_resources[0].find('/*! Source'):]
             return input_resources
 
         def as_html(notebook):
             conf = Config()
             conf.CustomHTMLExporter.preprocessors = [Sandbox]
             html_exporter = CustomHTMLExporter(config=conf)
 
             (body, resources) = html_exporter.from_notebook_node(notebook)
             header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
             js = MakoTemplate(r'''
             <!-- MathJax configuration -->
             <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
                 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
                 TeX: {
                     extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
                 },
                 tex2jax: {
                     inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                     displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
                     processEscapes: true,
                     processEnvironments: true
                 },
                 // Center justify equations in code and markdown cells. Elsewhere
                 // we use CSS to left justify single line equations in code cells.
                 displayAlign: 'center',
                 "HTML-CSS": {
                     styles: {'.MathJax_Display': {"margin": 0}},
                     linebreaks: { automatic: true },
                     availableFonts: ["STIX", "TeX"]
                 },
                 showMathMenu: false
             });
             </script>
             <!-- End of MathJax configuration -->
             <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
             ''').render(h=helpers)
 
             css = MakoTemplate(r'''
             <link rel="stylesheet" type="text/css" href="${h.asset('css/style-ipython.css', ver=ver)}" media="screen"/>
             ''').render(h=helpers, ver='ver1')
 
             body = '\n'.join([header, css, js, body])
             return body, resources
 
         notebook = nbformat.reads(source, as_version=4)
         (body, resources) = as_html(notebook)
         return body
 
 
 class RstTemplateRenderer(object):
 
     def __init__(self):
         base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
         rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
         self.template_store = TemplateLookup(
             directories=rst_template_dirs,
             input_encoding='utf-8',
             imports=['from rhodecode.lib import helpers as h'])
 
     def _get_template(self, templatename):
         return self.template_store.get_template(templatename)
 
     def render(self, template_name, **kwargs):
         template = self._get_template(template_name)
         return template.render(**kwargs)

rhodecode/lib/middleware/simplegit.py

@@ -1,156 +1,156 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleGit middleware for handling git protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 import os
 import re
 import logging
 import urllib.parse
 
 import rhodecode
 from rhodecode.lib import utils
 from rhodecode.lib import utils2
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 GIT_PROTO_PAT = re.compile(
     r'^/(.+)/(info/refs|info/lfs/(.+)|git-upload-pack|git-receive-pack)')
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def default_lfs_store():
     """
     Default lfs store location, it's consistent with Mercurials large file
     store which is in .cache/largefiles
     """
     from rhodecode.lib.vcs.backends.git import lfs_store
     user_home = os.path.expanduser("~")
     return lfs_store(user_home)
 
 
 class SimpleGit(simplevcs.SimpleVCS):
 
     SCM = 'git'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
         # for GIT LFS, and bare format strip .git suffix from names
         if repo_name.endswith('.git'):
             repo_name = repo_name[:-4]
         return repo_name
 
     def _get_lfs_action(self, path, request_method):
         """
         return an action based on LFS requests type.
         Those routes are handled inside vcsserver app.
 
         batch           -> POST to /info/lfs/objects/batch  => PUSH/PULL
                         batch is based on the `operation.
                         that could be download or upload, but those are only
                         instructions to fetch so we return pull always
 
         download        -> GET  to /info/lfs/{oid}          => PULL
         upload          -> PUT  to /info/lfs/{oid}          => PUSH
 
         verification    -> POST to /info/lfs/verify         => PULL
 
         """
 
         match_obj = GIT_LFS_PROTO_PAT.match(path)
         _parts = match_obj.groups()
         repo_name, path, operation = _parts
         log.debug(
             'LFS: detecting operation based on following '
             'data: %s, req_method:%s', _parts, request_method)
 
         if operation == 'verify':
             return 'pull'
         elif operation == 'objects/batch':
             # batch sends back instructions for API to dl/upl we report it
             # as pull
             if request_method == 'POST':
                 return 'pull'
 
         elif operation:
             # probably a OID, upload  is PUT, download a GET
             if request_method == 'GET':
                 return 'pull'
             else:
                 return 'push'
 
         # if default not found require push, as action
         return 'push'
 
     _ACTION_MAPPING = {
         'git-receive-pack': 'push',
         'git-upload-pack': 'pull',
     }
 
     def _get_action(self, environ):
         """
         Maps git request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'pull' to be safe.
 
         :param environ:
         """
         path = environ['PATH_INFO']
 
         if path.endswith('/info/refs'):
-            query = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING'])
+            query = urllib.parse.parse_qs(environ['QUERY_STRING'])
             service_cmd = query.get('service', [''])[0]
             return self._ACTION_MAPPING.get(service_cmd, 'pull')
 
         elif GIT_LFS_PROTO_PAT.match(environ['PATH_INFO']):
             return self._get_lfs_action(
                 environ['PATH_INFO'], environ['REQUEST_METHOD'])
 
         elif path.endswith('/git-receive-pack'):
             return 'push'
         elif path.endswith('/git-upload-pack'):
             return 'pull'
 
         return 'pull'
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_git_wsgi_app(
             repo_path, repo_name, config)
 
     def _create_config(self, extras, repo_name, scheme='http'):
         extras['git_update_server_info'] = utils2.str2bool(
             rhodecode.CONFIG.get('git_update_server_info'))
 
         config = utils.make_db_config(repo=repo_name)
         custom_store = config.get('vcs_git_lfs', 'store_location')
 
         extras['git_lfs_enabled'] = utils2.str2bool(
             config.get('vcs_git_lfs', 'enabled'))
         extras['git_lfs_store_path'] = custom_store or default_lfs_store()
         extras['git_lfs_http_scheme'] = scheme
         return extras

rhodecode/lib/middleware/simplehg.py

@@ -1,160 +1,159 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleHG middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 """
 
 import logging
 import urllib.parse
 import urllib.request, urllib.parse, urllib.error
 
 from rhodecode.lib import utils
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 class SimpleHg(simplevcs.SimpleVCS):
 
     SCM = 'hg'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = environ['PATH_INFO']
         if repo_name and repo_name.startswith('/'):
             # remove only the first leading /
             repo_name = repo_name[1:]
         return repo_name.rstrip('/')
 
     _ACTION_MAPPING = {
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
         'getbundle': 'pull',
         'stream_out': 'pull',
         'listkeys': 'pull',
         'between': 'pull',
         'branchmap': 'pull',
         'branches': 'pull',
         'clonebundles': 'pull',
         'capabilities': 'pull',
         'debugwireargs': 'pull',
         'heads': 'pull',
         'lookup': 'pull',
         'hello': 'pull',
         'known': 'pull',
 
         # largefiles
         'putlfile': 'push',
         'getlfile': 'pull',
         'statlfile': 'pull',
         'lheads': 'pull',
 
         # evolve
         'evoext_obshashrange_v1': 'pull',
         'evoext_obshash': 'pull',
         'evoext_obshash1': 'pull',
 
         'unbundle': 'push',
         'pushkey': 'push',
     }
 
     @classmethod
     def _get_xarg_headers(cls, environ):
         i = 1
         chunks = []  # gather chunks stored in multiple 'hgarg_N'
         while True:
             head = environ.get('HTTP_X_HGARG_{}'.format(i))
             if not head:
                 break
             i += 1
             chunks.append(urllib.parse.unquote_plus(head))
         full_arg = ''.join(chunks)
         pref = 'cmds='
         if full_arg.startswith(pref):
             # strip the cmds= header defining our batch commands
             full_arg = full_arg[len(pref):]
         cmds = full_arg.split(';')
         return cmds
 
     @classmethod
     def _get_batch_cmd(cls, environ):
         """
         Handle batch command send commands. Those are ';' separated commands
         sent by batch command that server needs to execute. We need to extract
         those, and map them to our ACTION_MAPPING to get all push/pull commands
         specified in the batch
         """
         default = 'push'
         batch_cmds = []
         try:
             cmds = cls._get_xarg_headers(environ)
             for pair in cmds:
                 parts = pair.split(' ', 1)
                 if len(parts) != 2:
                     continue
                 # entry should be in a format `key ARGS`
                 cmd, args = parts
                 action = cls._ACTION_MAPPING.get(cmd, default)
                 batch_cmds.append(action)
         except Exception:
             log.exception('Failed to extract batch commands operations')
 
         # in case we failed, (e.g malformed data) assume it's PUSH sub-command
         # for safety
         return batch_cmds or [default]
 
     def _get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'push' to be safe.
 
         :param environ:
         """
         default = 'push'
-        query = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING'],
-                                  keep_blank_values=True)
+        query = urllib.parse.parse_qs(environ['QUERY_STRING'], keep_blank_values=True)
 
         if 'cmd' in query:
             cmd = query['cmd'][0]
             if cmd == 'batch':
                 cmds = self._get_batch_cmd(environ)
                 if 'push' in cmds:
                     return 'push'
                 else:
                     return 'pull'
             return self._ACTION_MAPPING.get(cmd, default)
 
         return default
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
 
     def _create_config(self, extras, repo_name, scheme='http'):
         config = utils.make_db_config(repo=repo_name)
         config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
         return config.serialize()

rhodecode/lib/middleware/simplevcs.py

@@ -1,679 +1,679 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
+import io
 import logging
 import importlib
 from functools import wraps
-from io import StringIO
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = next(gen)
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
-        'pull-request/(?P<pr_id>\d+)/'  # pull request
+        'pull-request/(?P<pr_id>\\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
         realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
 
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = user_id
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
             data = environ['wsgi.input'].read()
-            environ['wsgi.input'] = StringIO(data)
+            environ['wsgi.input'] = io.StringIO(data)
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             direct_calls = True
 
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST, use_direct_calls=direct_calls, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/lib/middleware/vcs.py

@@ -1,284 +1,284 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import gzip
 import shutil
 import logging
 import tempfile
 import urllib.parse
 
 from webob.exc import HTTPNotFound
 
 import rhodecode
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
 from rhodecode.lib.middleware.simplehg import SimpleHg
 from rhodecode.lib.middleware.simplesvn import SimpleSvn
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 VCS_TYPE_KEY = '_rc_vcs_type'
 VCS_TYPE_SKIP = '_rc_vcs_skip'
 
 
 def is_git(environ):
     """
     Returns True if requests should be handled by GIT wsgi middleware
     """
     is_git_path = GIT_PROTO_PAT.match(environ['PATH_INFO'])
     log.debug(
         'request path: `%s` detected as GIT PROTOCOL %s', environ['PATH_INFO'],
         is_git_path is not None)
 
     return is_git_path
 
 
 def is_hg(environ):
     """
     Returns True if requests target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     is_hg_path = False
 
     http_accept = environ.get('HTTP_ACCEPT')
 
     if http_accept and http_accept.startswith('application/mercurial'):
-        query = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING'])
+        query = urllib.parse.parse_qs(environ['QUERY_STRING'])
         if 'cmd' in query:
             is_hg_path = True
 
     log.debug(
         'request path: `%s` detected as HG PROTOCOL %s', environ['PATH_INFO'],
         is_hg_path)
 
     return is_hg_path
 
 
 def is_svn(environ):
     """
     Returns True if requests target is Subversion server
     """
 
     http_dav = environ.get('HTTP_DAV', '')
     magic_path_segment = rhodecode.CONFIG.get(
         'rhodecode_subversion_magic_path', '/!svn')
     is_svn_path = (
         'subversion' in http_dav or
         magic_path_segment in environ['PATH_INFO']
         or environ['REQUEST_METHOD'] in ['PROPFIND', 'PROPPATCH']
     )
     log.debug(
         'request path: `%s` detected as SVN PROTOCOL %s', environ['PATH_INFO'],
         is_svn_path)
 
     return is_svn_path
 
 
 class GunzipMiddleware(object):
     """
     WSGI middleware that unzips gzip-encoded requests before
     passing on to the underlying application.
     """
 
     def __init__(self, application):
         self.app = application
 
     def __call__(self, environ, start_response):
         accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
 
         if b'gzip' in accepts_encoding_header:
             log.debug('gzip detected, now running gunzip wrapper')
             wsgi_input = environ['wsgi.input']
 
             if not hasattr(environ['wsgi.input'], 'seek'):
                 # The gzip implementation in the standard library of Python 2.x
                 # requires the '.seek()' and '.tell()' methods to be available
                 # on the input stream.  Read the data into a temporary file to
                 # work around this limitation.
 
                 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
                 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
                 wsgi_input.seek(0)
 
             environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
             # since we "Ungzipped" the content we say now it's no longer gzip
             # content encoding
             del environ['HTTP_CONTENT_ENCODING']
 
             # content length has changes ? or i'm not sure
             if 'CONTENT_LENGTH' in environ:
                 del environ['CONTENT_LENGTH']
         else:
             log.debug('content not gzipped, gzipMiddleware passing '
                       'request further')
         return self.app(environ, start_response)
 
 
 def is_vcs_call(environ):
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         return raw_type and raw_type != VCS_TYPE_SKIP
     return False
 
 
 def get_path_elem(route_path):
     if not route_path:
         return None
 
     cleaned_route_path = route_path.lstrip('/')
     if cleaned_route_path:
         cleaned_route_path_elems = cleaned_route_path.split('/')
         if cleaned_route_path_elems:
             return cleaned_route_path_elems[0]
     return None
 
 
 def detect_vcs_request(environ, backends):
     checks = {
         'hg': (is_hg, SimpleHg),
         'git': (is_git, SimpleGit),
         'svn': (is_svn, SimpleSvn),
     }
     handler = None
     # List of path views first chunk we don't do any checks
     white_list = [
         # e.g /_file_store/download
         '_file_store',
 
         # static files no detection
         '_static',
 
         # skip ops ping, status
         '_admin/ops/ping',
         '_admin/ops/status',
 
         # full channelstream connect should be VCS skipped
         '_admin/channelstream/connect',
     ]
 
     path_info = environ['PATH_INFO']
 
     path_elem = get_path_elem(path_info)
 
     if path_elem in white_list:
         log.debug('path `%s` in whitelist, skipping...', path_info)
         return handler
 
     path_url = path_info.lstrip('/')
     if path_url in white_list:
         log.debug('full url path `%s` in whitelist, skipping...', path_url)
         return handler
 
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         if raw_type == VCS_TYPE_SKIP:
             log.debug('got `skip` marker for vcs detection, skipping...')
             return handler
 
         _check, handler = checks.get(raw_type) or [None, None]
         if handler:
             log.debug('got handler:%s from environ', handler)
 
     if not handler:
         log.debug('request start: checking if request for `%s` is of VCS type in order: %s', path_elem, backends)
         for vcs_type in backends:
             vcs_check, _handler = checks[vcs_type]
             if vcs_check(environ):
                 log.debug('vcs handler found %s', _handler)
                 handler = _handler
                 break
 
     return handler
 
 
 class VCSMiddleware(object):
 
     def __init__(self, app, registry, config, appenlight_client):
         self.application = app
         self.registry = registry
         self.config = config
         self.appenlight_client = appenlight_client
         self.use_gzip = True
         # order in which we check the middlewares, based on vcs.backends config
         self.check_middlewares = config['vcs.backends']
 
     def vcs_config(self, repo_name=None):
         """
         returns serialized VcsSettings
         """
         try:
             return VcsSettingsModel(
                 repo=repo_name).get_ui_settings_as_config_obj()
         except Exception:
             pass
 
     def wrap_in_gzip_if_enabled(self, app, config):
         if self.use_gzip:
             app = GunzipMiddleware(app)
         return app
 
     def _get_handler_app(self, environ):
         app = None
         log.debug('VCSMiddleware: detecting vcs type.')
         handler = detect_vcs_request(environ, self.check_middlewares)
         if handler:
             app = handler(self.config, self.registry)
 
         return app
 
     def __call__(self, environ, start_response):
         # check if we handle one of interesting protocols, optionally extract
         # specific vcsSettings and allow changes of how things are wrapped
         vcs_handler = self._get_handler_app(environ)
         if vcs_handler:
             # translate the _REPO_ID into real repo NAME for usage
             # in middleware
             environ['PATH_INFO'] = vcs_handler._get_by_id(environ['PATH_INFO'])
 
             # Set acl, url and vcs repo names.
             vcs_handler.set_repo_names(environ)
 
             # register repo config back to the handler
             vcs_conf = self.vcs_config(vcs_handler.acl_repo_name)
             # maybe damaged/non existent settings. We still want to
             # pass that point to validate on is_valid_and_existing_repo
             # and return proper HTTP Code back to client
             if vcs_conf:
                 vcs_handler.repo_vcs_config = vcs_conf
 
             # check for type, presence in database and on filesystem
             if not vcs_handler.is_valid_and_existing_repo(
                     vcs_handler.acl_repo_name,
                     vcs_handler.base_path,
                     vcs_handler.SCM):
                 return HTTPNotFound()(environ, start_response)
 
             environ['REPO_NAME'] = vcs_handler.url_repo_name
 
             # Wrap handler in middlewares if they are enabled.
             vcs_handler = self.wrap_in_gzip_if_enabled(
                 vcs_handler, self.config)
             vcs_handler, _ = wrap_in_appenlight_if_enabled(
                 vcs_handler, self.config, self.appenlight_client)
 
             return vcs_handler(environ, start_response)
 
         return self.application(environ, start_response)

rhodecode/lib/pagination.py

@@ -1,1061 +1,1061 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (c) 2007-2012 Christoph Haas <[e-mail]>
 # NOTE: MIT license based code, backported and edited by RhodeCode GmbH
 
 """
 paginate: helps split up large collections into individual pages
 ================================================================
 
 What is pagination?
 ---------------------
 
 This module helps split large lists of items into pages. The user is shown one page at a time and
 can navigate to other pages. Imagine you are offering a company phonebook and let the user search
 the entries. The entire search result may contains 23 entries but you want to display no more than
 10 entries at once. The first page contains entries 1-10, the second 11-20 and the third 21-23.
 Each "Page" instance represents the items of one of these three pages.
 
 See the documentation of the "Page" class for more information.
 
 How do I use it?
 ------------------
 
 A page of items is represented by the *Page* object. A *Page* gets initialized with these arguments:
 
 - The collection of items to pick a range from. Usually just a list.
 - The page number you want to display. Default is 1: the first page.
 
 Now we can make up a collection and create a Page instance of it::
 
     # Create a sample collection of 1000 items
     >> my_collection = range(1000)
 
     # Create a Page object for the 3rd page (20 items per page is the default)
     >> my_page = Page(my_collection, page=3)
 
     # The page object can be printed as a string to get its details
     >> str(my_page)
     Page:
     Collection type:  <type 'range'>
     Current page:     3
     First item:       41
     Last item:        60
     First page:       1
     Last page:        50
     Previous page:    2
     Next page:        4
     Items per page:   20
     Number of items:  1000
     Number of pages:  50
 
     # Print a list of items on the current page
     >> my_page.items
     [40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
 
     # The *Page* object can be used as an iterator:
     >> for my_item in my_page: print(my_item)
     40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
 
     # The .pager() method returns an HTML fragment with links to surrounding pages.
     >> my_page.pager(url="http://example.org/foo/page=$page")
 
     <a href="http://example.org/foo/page=1">1</a>
     <a href="http://example.org/foo/page=2">2</a>
     3
     <a href="http://example.org/foo/page=4">4</a>
     <a href="http://example.org/foo/page=5">5</a>
     ..
     <a href="http://example.org/foo/page=50">50</a>'
 
     # Without the HTML it would just look like:
     # 1 2 [3] 4 5 .. 50
 
     # The pager can be customized:
     >> my_page.pager('$link_previous ~3~ $link_next (Page $page of $page_count)',
                      url="http://example.org/foo/page=$page")
 
     <a href="http://example.org/foo/page=2">&lt;</a>
     <a href="http://example.org/foo/page=1">1</a>
     <a href="http://example.org/foo/page=2">2</a>
     3
     <a href="http://example.org/foo/page=4">4</a>
     <a href="http://example.org/foo/page=5">5</a>
     <a href="http://example.org/foo/page=6">6</a>
     ..
     <a href="http://example.org/foo/page=50">50</a>
     <a href="http://example.org/foo/page=4">&gt;</a>
     (Page 3 of 50)
 
     # Without the HTML it would just look like:
     # 1 2 [3] 4 5 6 .. 50 > (Page 3 of 50)
 
     # The url argument to the pager method can be omitted when an url_maker is
     # given during instantiation:
     >> my_page = Page(my_collection, page=3,
                       url_maker=lambda p: "http://example.org/%s" % p)
     >> page.pager()
 
 There are some interesting parameters that customize the Page's behavior. See the documentation on
 ``Page`` and ``Page.pager()``.
 
 
 Notes
 -------
 
 Page numbers and item numbers start at 1. This concept has been used because users expect that the
 first page has number 1 and the first item on a page also has number 1. So if you want to use the
 page's items by their index number please note that you have to subtract 1.
 """
 
 import re
 import sys
 from string import Template
 from webhelpers2.html import literal
 
 # are we running at least python 3.x ?
 PY3 = sys.version_info[0] >= 3
 
 if PY3:
     unicode = str
 
 
 def make_html_tag(tag, text=None, **params):
     """Create an HTML tag string.
 
     tag
         The HTML tag to use (e.g. 'a', 'span' or 'div')
 
     text
         The text to enclose between opening and closing tag. If no text is specified then only
         the opening tag is returned.
 
     Example::
         make_html_tag('a', text="Hello", href="/another/page")
         -> <a href="/another/page">Hello</a>
 
     To use reserved Python keywords like "class" as a parameter prepend it with
     an underscore. Instead of "class='green'" use "_class='green'".
 
     Warning: Quotes and apostrophes are not escaped."""
     params_string = ""
 
     # Parameters are passed. Turn the dict into a string like "a=1 b=2 c=3" string.
     for key, value in sorted(params.items()):
         # Strip off a leading underscore from the attribute's key to allow attributes like '_class'
         # to be used as a CSS class specification instead of the reserved Python keyword 'class'.
         key = key.lstrip("_")
 
-        params_string += u' {0}="{1}"'.format(key, value)
+        params_string += ' {0}="{1}"'.format(key, value)
 
     # Create the tag string
-    tag_string = u"<{0}{1}>".format(tag, params_string)
+    tag_string = "<{0}{1}>".format(tag, params_string)
 
     # Add text and closing tag if required.
     if text:
-        tag_string += u"{0}</{1}>".format(text, tag)
+        tag_string += "{0}</{1}>".format(text, tag)
 
     return tag_string
 
 
 # Since the items on a page are mainly a list we subclass the "list" type
 class _Page(list):
     """A list/iterator representing the items on one page of a larger collection.
 
     An instance of the "Page" class is created from a _collection_ which is any
     list-like object that allows random access to its elements.
 
     The instance works as an iterator running from the first item to the last item on the given
     page. The Page.pager() method creates a link list allowing the user to go to other pages.
 
     A "Page" does not only carry the items on a certain page. It gives you additional information
     about the page in these "Page" object attributes:
 
     item_count
         Number of items in the collection
 
         **WARNING:** Unless you pass in an item_count, a count will be
         performed on the collection every time a Page instance is created.
 
     page
         Number of the current page
 
     items_per_page
         Maximal number of items displayed on a page
 
     first_page
         Number of the first page - usually 1 :)
 
     last_page
         Number of the last page
 
     previous_page
         Number of the previous page. If this is the first page it returns None.
 
     next_page
         Number of the next page. If this is the last page it returns None.
 
     page_count
         Number of pages
 
     items
         Sequence/iterator of items on the current page
 
     first_item
         Index of first item on the current page - starts with 1
 
     last_item
         Index of last item on the current page
     """
 
     def __init__(
             self,
             collection,
             page=1,
             items_per_page=20,
             item_count=None,
             wrapper_class=None,
             url_maker=None,
             bar_size=10,
             **kwargs
     ):
         """Create a "Page" instance.
 
         Parameters:
 
         collection
             Sequence representing the collection of items to page through.
 
         page
             The requested page number - starts with 1. Default: 1.
 
         items_per_page
             The maximal number of items to be displayed per page.
             Default: 20.
 
         item_count (optional)
             The total number of items in the collection - if known.
             If this parameter is not given then the paginator will count
             the number of elements in the collection every time a "Page"
             is created. Giving this parameter will speed up things. In a busy
             real-life application you may want to cache the number of items.
 
         url_maker (optional)
             Callback to generate the URL of other pages, given its numbers.
             Must accept one int parameter and return a URI string.
 
         bar_size
             maximum size of rendered pages numbers within radius
 
         """
         if collection is not None:
             if wrapper_class is None:
                 # Default case. The collection is already a list-type object.
                 self.collection = collection
             else:
                 # Special case. A custom wrapper class is used to access elements of the collection.
                 self.collection = wrapper_class(collection)
         else:
             self.collection = []
 
         self.collection_type = type(collection)
 
         if url_maker is not None:
             self.url_maker = url_maker
         else:
             self.url_maker = self._default_url_maker
         self.bar_size = bar_size
         # Assign kwargs to self
         self.kwargs = kwargs
 
         # The self.page is the number of the current page.
         # The first page has the number 1!
         try:
             self.page = int(page)  # make it int() if we get it as a string
         except (ValueError, TypeError):
             self.page = 1
         # normally page should be always at least 1 but the original maintainer
         # decided that for empty collection and empty page it can be...0? (based on tests)
         # preserving behavior for BW compat
         if self.page < 1:
             self.page = 1
 
         self.items_per_page = items_per_page
 
         # We subclassed "list" so we need to call its init() method
         # and fill the new list with the items to be displayed on the page.
         # We use list() so that the items on the current page are retrieved
         # only once. In an SQL context that could otherwise lead to running the
         # same SQL query every time items would be accessed.
         # We do this here, prior to calling len() on the collection so that a
         # wrapper class can execute a query with the knowledge of what the
         # slice will be (for efficiency) and, in the same query, ask for the
         # total number of items and only execute one query.
 
         try:
             first = (self.page - 1) * items_per_page
             last = first + items_per_page
             self.items = list(self.collection[first:last])
         except TypeError as err:
             raise TypeError(
                 f"Your collection of type {type(self.collection)} cannot be handled "
                 f"by paginate. ERROR:{err}"
             )
 
         # Unless the user tells us how many items the collections has
         # we calculate that ourselves.
         if item_count is not None:
             self.item_count = item_count
         else:
             self.item_count = len(self.collection)
 
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
             self.page_count = ((self.item_count - 1) // self.items_per_page) + 1
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of valid pages
             if self.page > self.last_page:
                 self.page = self.last_page
             elif self.page < self.first_page:
                 self.page = self.first_page
 
             # Note: the number of items on this page can be less than
             #       items_per_page if the last page is not full
             self.first_item = (self.page - 1) * items_per_page + 1
             self.last_item = min(self.first_item + items_per_page - 1, self.item_count)
 
             # Links to previous and next page
             if self.page > self.first_page:
                 self.previous_page = self.page - 1
             else:
                 self.previous_page = None
 
             if self.page < self.last_page:
                 self.next_page = self.page + 1
             else:
                 self.next_page = None
 
         # No items available
         else:
             self.first_page = None
             self.page_count = 0
             self.last_page = None
             self.first_item = None
             self.last_item = None
             self.previous_page = None
             self.next_page = None
             self.items = []
 
         # This is a subclass of the 'list' type. Initialise the list now.
         list.__init__(self, self.items)
 
     def __str__(self):
         return (
             "Page:\n"
             "Collection type:        {0.collection_type}\n"
             "Current page:           {0.page}\n"
             "First item:             {0.first_item}\n"
             "Last item:              {0.last_item}\n"
             "First page:             {0.first_page}\n"
             "Last page:              {0.last_page}\n"
             "Previous page:          {0.previous_page}\n"
             "Next page:              {0.next_page}\n"
             "Items per page:         {0.items_per_page}\n"
             "Total number of items:  {0.item_count}\n"
             "Number of pages:        {0.page_count}\n"
         ).format(self)
 
     def __repr__(self):
         return "<paginate.Page: Page {0}/{1}>".format(self.page, self.page_count)
 
     def pager(
             self,
             tmpl_format="~2~",
             url=None,
             show_if_single_page=False,
             separator=" ",
             symbol_first="&lt;&lt;",
             symbol_last="&gt;&gt;",
             symbol_previous="&lt;",
             symbol_next="&gt;",
             link_attr=None,
             curpage_attr=None,
             dotdot_attr=None,
             link_tag=None,
     ):
         """
         Return string with links to other pages (e.g. '1 .. 5 6 7 [8] 9 10 11 .. 50').
 
         tmpl_format:
             Format string that defines how the pager is rendered. The string
             can contain the following $-tokens that are substituted by the
             string.Template module:
 
             - $first_page: number of first reachable page
             - $last_page: number of last reachable page
             - $page: number of currently selected page
             - $page_count: number of reachable pages
             - $items_per_page: maximal number of items per page
             - $first_item: index of first item on the current page
             - $last_item: index of last item on the current page
             - $item_count: total number of items
             - $link_first: link to first page (unless this is first page)
             - $link_last: link to last page (unless this is last page)
             - $link_previous: link to previous page (unless this is first page)
             - $link_next: link to next page (unless this is last page)
 
             To render a range of pages the token '~3~' can be used. The
             number sets the radius of pages around the current page.
             Example for a range with radius 3:
 
             '1 .. 5 6 7 [8] 9 10 11 .. 50'
 
             Default: '~2~'
 
         url
             The URL that page links will point to. Make sure it contains the string
             $page which will be replaced by the actual page number.
             Must be given unless a url_maker is specified to __init__, in which
             case this parameter is ignored.
 
         symbol_first
             String to be displayed as the text for the $link_first link above.
 
             Default: '&lt;&lt;' (<<)
 
         symbol_last
             String to be displayed as the text for the $link_last link above.
 
             Default: '&gt;&gt;' (>>)
 
         symbol_previous
             String to be displayed as the text for the $link_previous link above.
 
             Default: '&lt;' (<)
 
         symbol_next
             String to be displayed as the text for the $link_next link above.
 
             Default: '&gt;' (>)
 
         separator:
             String that is used to separate page links/numbers in the above range of pages.
 
             Default: ' '
 
         show_if_single_page:
             if True the navigator will be shown even if there is only one page.
 
             Default: False
 
         link_attr (optional)
             A dictionary of attributes that get added to A-HREF links pointing to other pages. Can
             be used to define a CSS style or class to customize the look of links.
 
             Example: { 'style':'border: 1px solid green' }
             Example: { 'class':'pager_link' }
 
         curpage_attr (optional)
             A dictionary of attributes that get added to the current page number in the pager (which
             is obviously not a link). If this dictionary is not empty then the elements will be
             wrapped in a SPAN tag with the given attributes.
 
             Example: { 'style':'border: 3px solid blue' }
             Example: { 'class':'pager_curpage' }
 
         dotdot_attr (optional)
             A dictionary of attributes that get added to the '..' string in the pager (which is
             obviously not a link). If this dictionary is not empty then the elements will be wrapped
             in a SPAN tag with the given attributes.
 
             Example: { 'style':'color: #808080' }
             Example: { 'class':'pager_dotdot' }
 
         link_tag (optional)
             A callable that accepts single argument `page` (page link information)
             and generates string with html that represents the link for specific page.
             Page objects are supplied from `link_map()` so the keys are the same.
 
 
         """
         link_attr = link_attr or {}
         curpage_attr = curpage_attr or {}
         dotdot_attr = dotdot_attr or {}
         self.curpage_attr = curpage_attr
         self.separator = separator
         self.link_attr = link_attr
         self.dotdot_attr = dotdot_attr
         self.url = url
         self.link_tag = link_tag or self.default_link_tag
 
         # Don't show navigator if there is no more than one page
         if self.page_count == 0 or (self.page_count == 1 and not show_if_single_page):
             return ""
 
         regex_res = re.search(r"~(\d+)~", tmpl_format)
         if regex_res:
             radius = regex_res.group(1)
         else:
             radius = 2
 
         self.radius = int(radius)
         link_map = self.link_map(
             tmpl_format=tmpl_format,
             url=url,
             show_if_single_page=show_if_single_page,
             separator=separator,
             symbol_first=symbol_first,
             symbol_last=symbol_last,
             symbol_previous=symbol_previous,
             symbol_next=symbol_next,
             link_attr=link_attr,
             curpage_attr=curpage_attr,
             dotdot_attr=dotdot_attr,
             link_tag=link_tag,
         )
         links_markup = self._range(link_map, self.radius)
 
         # Replace ~...~ in token tmpl_format by range of pages
         result = re.sub(r"~(\d+)~", links_markup, tmpl_format)
 
         link_first = (
             self.page > self.first_page and self.link_tag(link_map["first_page"]) or ""
         )
         link_last = (
             self.page < self.last_page and self.link_tag(link_map["last_page"]) or ""
         )
         link_previous = (
             self.previous_page and self.link_tag(link_map["previous_page"]) or ""
         )
         link_next = self.next_page and self.link_tag(link_map["next_page"]) or ""
         # Interpolate '$' variables
         result = Template(result).safe_substitute(
             {
                 "first_page": self.first_page,
                 "last_page": self.last_page,
                 "page": self.page,
                 "page_count": self.page_count,
                 "items_per_page": self.items_per_page,
                 "first_item": self.first_item,
                 "last_item": self.last_item,
                 "item_count": self.item_count,
                 "link_first": link_first,
                 "link_last": link_last,
                 "link_previous": link_previous,
                 "link_next": link_next,
             }
         )
 
         return result
 
     def _get_edges(self, cur_page, max_page, items):
         cur_page = int(cur_page)
-        edge = (items / 2) + 1
+        edge = (items // 2) + 1
         if cur_page <= edge:
-            radius = max(items / 2, items - cur_page)
+            radius = max(items // 2, items - cur_page)
         elif (max_page - cur_page) < edge:
             radius = (items - 1) - (max_page - cur_page)
         else:
-            radius = (items / 2) - 1
+            radius = (items // 2) - 1
 
         left = max(1, (cur_page - radius))
         right = min(max_page, cur_page + radius)
         return left, right
 
     def link_map(
             self,
             tmpl_format="~2~",
             url=None,
             show_if_single_page=False,
             separator=" ",
             symbol_first="&lt;&lt;",
             symbol_last="&gt;&gt;",
             symbol_previous="&lt;",
             symbol_next="&gt;",
             link_attr=None,
             curpage_attr=None,
             dotdot_attr=None,
             link_tag=None
     ):
         """ Return map with links to other pages if default pager() function is not suitable solution.
         tmpl_format:
             Format string that defines how the pager would be normally rendered rendered. Uses same arguments as pager()
             method, but returns a simple dictionary in form of:
             {'current_page': {'attrs': {},
                                      'href': 'http://example.org/foo/page=1',
                                      'value': 1},
                     'first_page': {'attrs': {},
                                    'href': 'http://example.org/foo/page=1',
                                    'type': 'first_page',
                                    'value': 1},
                     'last_page': {'attrs': {},
                                   'href': 'http://example.org/foo/page=8',
                                   'type': 'last_page',
                                   'value': 8},
                     'next_page': {'attrs': {}, 'href': 'HREF', 'type': 'next_page', 'value': 2},
                     'previous_page': None,
                     'range_pages': [{'attrs': {},
                                      'href': 'http://example.org/foo/page=1',
                                      'type': 'current_page',
                                      'value': 1},
                                      ....
                                     {'attrs': {}, 'href': '', 'type': 'span', 'value': '..'}]}
 
 
             The string can contain the following $-tokens that are substituted by the
             string.Template module:
 
             - $first_page: number of first reachable page
             - $last_page: number of last reachable page
             - $page: number of currently selected page
             - $page_count: number of reachable pages
             - $items_per_page: maximal number of items per page
             - $first_item: index of first item on the current page
             - $last_item: index of last item on the current page
             - $item_count: total number of items
             - $link_first: link to first page (unless this is first page)
             - $link_last: link to last page (unless this is last page)
             - $link_previous: link to previous page (unless this is first page)
             - $link_next: link to next page (unless this is last page)
 
             To render a range of pages the token '~3~' can be used. The
             number sets the radius of pages around the current page.
             Example for a range with radius 3:
 
             '1 .. 5 6 7 [8] 9 10 11 .. 50'
 
             Default: '~2~'
 
         url
             The URL that page links will point to. Make sure it contains the string
             $page which will be replaced by the actual page number.
             Must be given unless a url_maker is specified to __init__, in which
             case this parameter is ignored.
 
         symbol_first
             String to be displayed as the text for the $link_first link above.
 
             Default: '&lt;&lt;' (<<)
 
         symbol_last
             String to be displayed as the text for the $link_last link above.
 
             Default: '&gt;&gt;' (>>)
 
         symbol_previous
             String to be displayed as the text for the $link_previous link above.
 
             Default: '&lt;' (<)
 
         symbol_next
             String to be displayed as the text for the $link_next link above.
 
             Default: '&gt;' (>)
 
         separator:
             String that is used to separate page links/numbers in the above range of pages.
 
             Default: ' '
 
         show_if_single_page:
             if True the navigator will be shown even if there is only one page.
 
             Default: False
 
         link_attr (optional)
             A dictionary of attributes that get added to A-HREF links pointing to other pages. Can
             be used to define a CSS style or class to customize the look of links.
 
             Example: { 'style':'border: 1px solid green' }
             Example: { 'class':'pager_link' }
 
         curpage_attr (optional)
             A dictionary of attributes that get added to the current page number in the pager (which
             is obviously not a link). If this dictionary is not empty then the elements will be
             wrapped in a SPAN tag with the given attributes.
 
             Example: { 'style':'border: 3px solid blue' }
             Example: { 'class':'pager_curpage' }
 
         dotdot_attr (optional)
             A dictionary of attributes that get added to the '..' string in the pager (which is
             obviously not a link). If this dictionary is not empty then the elements will be wrapped
             in a SPAN tag with the given attributes.
 
             Example: { 'style':'color: #808080' }
             Example: { 'class':'pager_dotdot' }
         """
         link_attr = link_attr or {}
         curpage_attr = curpage_attr or {}
         dotdot_attr = dotdot_attr or {}
         self.curpage_attr = curpage_attr
         self.separator = separator
         self.link_attr = link_attr
         self.dotdot_attr = dotdot_attr
         self.url = url
 
         regex_res = re.search(r"~(\d+)~", tmpl_format)
         if regex_res:
             radius = regex_res.group(1)
         else:
             radius = 2
 
         self.radius = int(radius)
 
         # Compute the first and last page number within the radius
         # e.g. '1 .. 5 6 [7] 8 9 .. 12'
         # -> leftmost_page  = 5
         # -> rightmost_page = 9
         leftmost_page, rightmost_page = self._get_edges(
             self.page, self.last_page, (self.radius * 2) + 1)
 
         nav_items = {
             "first_page": None,
             "last_page": None,
             "previous_page": None,
             "next_page": None,
             "current_page": None,
             "radius": self.radius,
             "range_pages": [],
         }
 
         if leftmost_page is None or rightmost_page is None:
             return nav_items
 
         nav_items["first_page"] = {
             "type": "first_page",
-            "value": unicode(symbol_first),
+            "value": str(symbol_first),
             "attrs": self.link_attr,
             "number": self.first_page,
             "href": self.url_maker(self.first_page),
         }
 
         # Insert dots if there are pages between the first page
         # and the currently displayed page range
         if leftmost_page - self.first_page > 1:
             # Wrap in a SPAN tag if dotdot_attr is set
             nav_items["range_pages"].append(
                 {
                     "type": "span",
                     "value": "..",
                     "attrs": self.dotdot_attr,
                     "href": "",
                     "number": None,
                 }
             )
 
         for this_page in range(leftmost_page, rightmost_page + 1):
             # Highlight the current page number and do not use a link
             if this_page == self.page:
                 # Wrap in a SPAN tag if curpage_attr is set
                 nav_items["range_pages"].append(
                     {
                         "type": "current_page",
-                        "value": unicode(this_page),
+                        "value": str(this_page),
                         "number": this_page,
                         "attrs": self.curpage_attr,
                         "href": self.url_maker(this_page),
                     }
                 )
                 nav_items["current_page"] = {
                     "value": this_page,
                     "attrs": self.curpage_attr,
                     "type": "current_page",
                     "href": self.url_maker(this_page),
                 }
             # Otherwise create just a link to that page
             else:
                 nav_items["range_pages"].append(
                     {
                         "type": "page",
-                        "value": unicode(this_page),
+                        "value": str(this_page),
                         "number": this_page,
                         "attrs": self.link_attr,
                         "href": self.url_maker(this_page),
                     }
                 )
 
         # Insert dots if there are pages between the displayed
         # page numbers and the end of the page range
         if self.last_page - rightmost_page > 1:
             # Wrap in a SPAN tag if dotdot_attr is set
             nav_items["range_pages"].append(
                 {
                     "type": "span",
                     "value": "..",
                     "attrs": self.dotdot_attr,
                     "href": "",
                     "number": None,
                 }
             )
 
         # Create a link to the very last page (unless we are on the last
         # page or there would be no need to insert '..' spacers)
         nav_items["last_page"] = {
             "type": "last_page",
-            "value": unicode(symbol_last),
+            "value": str(symbol_last),
             "attrs": self.link_attr,
             "href": self.url_maker(self.last_page),
             "number": self.last_page,
         }
 
         nav_items["previous_page"] = {
             "type": "previous_page",
-            "value": unicode(symbol_previous),
+            "value": str(symbol_previous),
             "attrs": self.link_attr,
             "number": self.previous_page or self.first_page,
             "href": self.url_maker(self.previous_page or self.first_page),
         }
 
         nav_items["next_page"] = {
             "type": "next_page",
-            "value": unicode(symbol_next),
+            "value": str(symbol_next),
             "attrs": self.link_attr,
             "number": self.next_page or self.last_page,
             "href": self.url_maker(self.next_page or self.last_page),
         }
 
         return nav_items
 
     def _range(self, link_map, radius):
         """
         Return range of linked pages to substitute placeholder in pattern
         """
         # Compute the first and last page number within the radius
         # e.g. '1 .. 5 6 [7] 8 9 .. 12'
         # -> leftmost_page  = 5
         # -> rightmost_page = 9
         leftmost_page, rightmost_page = self._get_edges(
             self.page, self.last_page, (radius * 2) + 1)
 
         nav_items = []
         # Create a link to the first page (unless we are on the first page
         # or there would be no need to insert '..' spacers)
         if self.first_page and self.page != self.first_page and self.first_page < leftmost_page:
             page = link_map["first_page"].copy()
-            page["value"] = unicode(page["number"])
+            page["value"] = str(page["number"])
             nav_items.append(self.link_tag(page))
 
         for item in link_map["range_pages"]:
             nav_items.append(self.link_tag(item))
 
         # Create a link to the very last page (unless we are on the last
         # page or there would be no need to insert '..' spacers)
         if self.last_page and self.page != self.last_page and rightmost_page < self.last_page:
             page = link_map["last_page"].copy()
-            page["value"] = unicode(page["number"])
+            page["value"] = str(page["number"])
             nav_items.append(self.link_tag(page))
 
         return self.separator.join(nav_items)
 
     def _default_url_maker(self, page_number):
         if self.url is None:
             raise Exception(
                 "You need to specify a 'url' parameter containing a '$page' placeholder."
             )
 
         if "$page" not in self.url:
             raise Exception("The 'url' parameter must contain a '$page' placeholder.")
 
-        return self.url.replace("$page", unicode(page_number))
+        return self.url.replace("$page", str(page_number))
 
     @staticmethod
     def default_link_tag(item):
         """
         Create an A-HREF tag that points to another page.
         """
         text = item["value"]
         target_url = item["href"]
 
         if not item["href"] or item["type"] in ("span", "current_page"):
             if item["attrs"]:
                 text = make_html_tag("span", **item["attrs"]) + text + "</span>"
             return text
 
         return make_html_tag("a", text=text, href=target_url, **item["attrs"])
 
 # Below is RhodeCode custom code
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 PAGE_FORMAT = '$link_previous ~3~ $link_next'
 
 
 class SqlalchemyOrmWrapper(object):
     """Wrapper class to access elements of a collection."""
 
     def __init__(self, pager, collection):
         self.pager = pager
         self.collection = collection
 
     def __getitem__(self, range):
         # Return a range of objects of an sqlalchemy.orm.query.Query object
         return self.collection[range]
 
     def __len__(self):
         # support empty types, without actually making a query.
         if self.collection is None or self.collection == []:
             return 0
 
         # Count the number of objects in an sqlalchemy.orm.query.Query object
         return self.collection.count()
 
 
 class CustomPager(_Page):
 
     @staticmethod
     def disabled_link_tag(item):
         """
         Create an A-HREF tag that is disabled
         """
         text = item['value']
         attrs = item['attrs'].copy()
         attrs['class'] = 'disabled ' + attrs['class']
 
         return make_html_tag('a', text=text, **attrs)
 
     def render(self):
         # Don't show navigator if there is no more than one page
         if self.page_count == 0:
             return ""
 
         self.link_tag = self.default_link_tag
 
         link_map = self.link_map(
             tmpl_format=PAGE_FORMAT, url=None,
             show_if_single_page=False, separator=' ',
             symbol_first='<<', symbol_last='>>',
             symbol_previous='<', symbol_next='>',
             link_attr={'class': 'pager_link'},
             curpage_attr={'class': 'pager_curpage'},
             dotdot_attr={'class': 'pager_dotdot'})
 
         links_markup = self._range(link_map, self.radius)
 
         link_first = (
             self.page > self.first_page and self.link_tag(link_map['first_page']) or ''
         )
         link_last = (
             self.page < self.last_page and self.link_tag(link_map['last_page']) or ''
         )
 
         link_previous = (
             self.previous_page and self.link_tag(link_map['previous_page'])
             or self.disabled_link_tag(link_map['previous_page'])
         )
         link_next = (
             self.next_page and self.link_tag(link_map['next_page'])
             or self.disabled_link_tag(link_map['next_page'])
         )
 
         # Interpolate '$' variables
         # Replace ~...~ in token tmpl_format by range of pages
         result = re.sub(r"~(\d+)~", links_markup, PAGE_FORMAT)
         result = Template(result).safe_substitute(
             {
                 "links": links_markup,
                 "first_page": self.first_page,
                 "last_page": self.last_page,
                 "page": self.page,
                 "page_count": self.page_count,
                 "items_per_page": self.items_per_page,
                 "first_item": self.first_item,
                 "last_item": self.last_item,
                 "item_count": self.item_count,
                 "link_first": link_first,
                 "link_last": link_last,
                 "link_previous": link_previous,
                 "link_next": link_next,
             }
         )
 
         return literal(result)
 
 
 class Page(CustomPager):
     """
     Custom pager to match rendering style with paginator
     """
 
     def __init__(self, collection, page=1, items_per_page=20, item_count=None,
                  url_maker=None, **kwargs):
         """
         Special type of pager. We intercept collection to wrap it in our custom
         logic instead of using wrapper_class
         """
 
         super(Page, self).__init__(collection=collection, page=page,
                                    items_per_page=items_per_page, item_count=item_count,
                                    wrapper_class=None, url_maker=url_maker, **kwargs)
 
 
 class SqlPage(CustomPager):
     """
     Custom pager to match rendering style with paginator
     """
 
     def __init__(self, collection, page=1, items_per_page=20, item_count=None,
                  url_maker=None, **kwargs):
         """
         Special type of pager. We intercept collection to wrap it in our custom
         logic instead of using wrapper_class
         """
         collection = SqlalchemyOrmWrapper(self, collection)
 
         super(SqlPage, self).__init__(collection=collection, page=page,
                                       items_per_page=items_per_page, item_count=item_count,
                                       wrapper_class=None, url_maker=url_maker, **kwargs)
 
 
 class RepoCommitsWrapper(object):
     """Wrapper class to access elements of a collection."""
 
     def __init__(self, pager, collection):
         self.pager = pager
         self.collection = collection
 
     def __getitem__(self, range):
         cur_page = self.pager.page
         items_per_page = self.pager.items_per_page
         first_item = max(0, (len(self.collection) - (cur_page * items_per_page)))
         last_item = ((len(self.collection) - 1) - items_per_page * (cur_page - 1))
         return reversed(list(self.collection[first_item:last_item + 1]))
 
     def __len__(self):
         return len(self.collection)
 
 
 class RepoPage(CustomPager):
     """
     Create a "RepoPage" instance. special pager for paging repository
     """
 
     def __init__(self, collection, page=1, items_per_page=20, item_count=None,
                  url_maker=None, **kwargs):
         """
         Special type of pager. We intercept collection to wrap it in our custom
         logic instead of using wrapper_class
         """
         collection = RepoCommitsWrapper(self, collection)
         super(RepoPage, self).__init__(collection=collection, page=page,
                                        items_per_page=items_per_page, item_count=item_count,
                                        wrapper_class=None, url_maker=url_maker, **kwargs)

rhodecode/lib/user_sessions.py

@@ -1,264 +1,264 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import time
 import datetime
 import dateutil
 import pickle
 
 from rhodecode.model.db import DbSession, Session
 
 
 class CleanupCommand(Exception):
     pass
 
 
 class BaseAuthSessions(object):
     SESSION_TYPE = None
     NOT_AVAILABLE = 'NOT AVAILABLE'
 
     def __init__(self, config):
         session_conf = {}
         for k, v in config.items():
             if k.startswith('beaker.session'):
                 session_conf[k] = v
         self.config = session_conf
 
     def get_count(self):
         raise NotImplementedError
 
     def get_expired_count(self, older_than_seconds=None):
         raise NotImplementedError
 
     def clean_sessions(self, older_than_seconds=None):
         raise NotImplementedError
 
     def _seconds_to_date(self, seconds):
         return datetime.datetime.utcnow() - dateutil.relativedelta.relativedelta(
             seconds=seconds)
 
 
 class DbAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'ext:database'
 
     def get_count(self):
         return DbSession.query().count()
 
     def get_expired_count(self, older_than_seconds=None):
         expiry_date = self._seconds_to_date(older_than_seconds)
         return DbSession.query().filter(DbSession.accessed < expiry_date).count()
 
     def clean_sessions(self, older_than_seconds=None):
         expiry_date = self._seconds_to_date(older_than_seconds)
         to_remove = DbSession.query().filter(DbSession.accessed < expiry_date).count()
         DbSession.query().filter(DbSession.accessed < expiry_date).delete()
         Session().commit()
         return to_remove
 
 
 class FileAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'file sessions'
 
     def _get_sessions_dir(self):
         data_dir = self.config.get('beaker.session.data_dir')
         return data_dir
 
     def _count_on_filesystem(self, path, older_than=0, callback=None):
         value = dict(percent=0, used=0, total=0, items=0, callbacks=0,
                      path=path, text='')
         items_count = 0
         used = 0
         callbacks = 0
         cur_time = time.time()
         for root, dirs, files in os.walk(path):
             for f in files:
                 final_path = os.path.join(root, f)
                 try:
                     mtime = os.stat(final_path).st_mtime
                     if (cur_time - mtime) > older_than:
                         items_count += 1
                         if callback:
                             callback_res = callback(final_path)
                             callbacks += 1
                         else:
                             used += os.path.getsize(final_path)
                 except OSError:
                     pass
         value.update({
             'percent': 100,
             'used': used,
             'total': used,
             'items': items_count,
             'callbacks': callbacks
         })
         return value
 
     def get_count(self):
         try:
             sessions_dir = self._get_sessions_dir()
             items_count = self._count_on_filesystem(sessions_dir)['items']
         except Exception:
             items_count = self.NOT_AVAILABLE
         return items_count
 
     def get_expired_count(self, older_than_seconds=0):
         try:
             sessions_dir = self._get_sessions_dir()
             items_count = self._count_on_filesystem(
                 sessions_dir, older_than=older_than_seconds)['items']
         except Exception:
             items_count = self.NOT_AVAILABLE
         return items_count
 
     def clean_sessions(self, older_than_seconds=0):
         # find . -mtime +60 -exec rm {} \;
 
         sessions_dir = self._get_sessions_dir()
 
         def remove_item(path):
             os.remove(path)
 
         stats = self._count_on_filesystem(
             sessions_dir, older_than=older_than_seconds,
             callback=remove_item)
         return stats['callbacks']
 
 
 class MemcachedAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'ext:memcached'
     _key_regex = re.compile(r'ITEM (.*_session) \[(.*); (.*)\]')
 
     def _get_client(self):
         import memcache
         client = memcache.Client([self.config.get('beaker.session.url')])
         return client
 
     def _get_telnet_client(self, host, port):
         import telnetlib
         client = telnetlib.Telnet(host, port, None)
         return client
 
     def _run_telnet_cmd(self, client, cmd):
         client.write("%s\n" % cmd)
         return client.read_until('END')
 
     def key_details(self, client, slab_ids, limit=100):
         """  Return a list of tuples containing keys and details """
         cmd = 'stats cachedump %s %s'
         for slab_id in slab_ids:
             for key in self._key_regex.finditer(
                     self._run_telnet_cmd(client, cmd % (slab_id, limit))):
                 yield key
 
     def get_count(self):
         client = self._get_client()
         count = self.NOT_AVAILABLE
         try:
             slabs = []
             for server, slabs_data in client.get_slabs():
-                slabs.extend(slabs_data.keys())
+                slabs.extend(list(slabs_data.keys()))
 
             host, port = client.servers[0].address
             telnet_client = self._get_telnet_client(host, port)
             keys = self.key_details(telnet_client, slabs)
             count = 0
             for _k in keys:
                 count += 1
         except Exception:
             return count
 
         return count
 
     def get_expired_count(self, older_than_seconds=None):
         return self.NOT_AVAILABLE
 
     def clean_sessions(self, older_than_seconds=None):
         raise CleanupCommand('Cleanup for this session type not yet available')
 
 
 class RedisAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'ext:redis'
 
     def _get_client(self):
         import redis
         args = {
             'socket_timeout': 60,
             'url': self.config.get('beaker.session.url')
         }
 
         client = redis.StrictRedis.from_url(**args)
         return client
 
     def get_count(self):
         client = self._get_client()
         return len(client.keys('beaker_cache:*'))
 
     def get_expired_count(self, older_than_seconds=None):
         expiry_date = self._seconds_to_date(older_than_seconds)
         return self.NOT_AVAILABLE
 
     def clean_sessions(self, older_than_seconds=None):
         client = self._get_client()
         expiry_time = time.time() - older_than_seconds
         deleted_keys = 0
         for key in client.keys('beaker_cache:*'):
             data = client.get(key)
             if data:
                 json_data = pickle.loads(data)
                 try:
                     accessed_time = json_data['_accessed_time']
                 except KeyError:
                     accessed_time = 0
                 if accessed_time < expiry_time:
                     client.delete(key)
                     deleted_keys += 1
 
         return deleted_keys
 
 
 class MemoryAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'memory'
 
     def get_count(self):
         return self.NOT_AVAILABLE
 
     def get_expired_count(self, older_than_seconds=None):
         return self.NOT_AVAILABLE
 
     def clean_sessions(self, older_than_seconds=None):
         raise CleanupCommand('Cleanup for this session type not yet available')
 
 
 def get_session_handler(session_type):
     types = {
         'file': FileAuthSessions,
         'ext:memcached': MemcachedAuthSessions,
         'ext:redis': RedisAuthSessions,
         'ext:database': DbAuthSessions,
         'memory': MemoryAuthSessions
     }
 
     try:
         return types[session_type]
     except KeyError:
         raise ValueError(
             'This type {} is not supported'.format(session_type))

rhodecode/lib/utils.py

@@ -1,799 +1,799 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import json
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
 import warnings
 import hashlib
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, remove_formatting
 from mako import exceptions
 from pyramid.threadlocal import get_current_registry
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, get_current_rhodecode_user, md5, sha1)
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape('`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile('[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile('[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile('[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
     replacement_char = '-'
 
     slug = remove_formatting(value)
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub('[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo.repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = _get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def _get_dirpaths(p):
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
-                u"Ignoring path %s since it cannot be decoded into unicode.",
+                "Ignoring path %s since it cannot be decoded into unicode.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group, False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check, we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
-        ok = raw_input(prompt)
+        ok = eval(input(prompt))
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
     for setting in ui_settings:
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(map(lambda s: '[{}] {}={}'.format(*s), ui_data)))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in app_settings.items():
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
     from rhodecode.model.settings import SettingsModel
     paths_ui = SettingsModel().get_ui_by_section_and_key('paths', '/')
     return safe_str(paths_ui.ui_value)
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks()
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    initial_repo_list.keys() if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         nested_sort = lambda gr: len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warn('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     import rc_testdata
 
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True})
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(auth_user.password) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return hashlib.sha256(':'.join(uuid_list)).hexdigest()
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = 'SUBJECT FROM: {}'.format(socket.gethostname())
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)

rhodecode/lib/utils2.py

@@ -1,1045 +1,1045 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Some simple helper functions
 """
 
 import collections
 import datetime
 import dateutil.relativedelta
 import logging
 import re
 import sys
 import time
 import urllib.request, urllib.parse, urllib.error
 import urlobject
 import uuid
 import getpass
 import socket
 import errno
 import random
 from functools import update_wrapper, partial, wraps
 from contextlib import closing
 
 import pygments.lexers
 import sqlalchemy
 import sqlalchemy.engine.url
 import sqlalchemy.exc
 import sqlalchemy.sql
 import webob
 import pyramid.threadlocal
 from pyramid.settings import asbool
 
 import rhodecode
 from rhodecode.translation import _, _pluralize
 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
 from rhodecode.lib.hash_utils import md5, md5_safe, sha1, sha1_safe
 from rhodecode.lib.type_utils import aslist, str2bool
 from functools import reduce
 
 #TODO: there's no longer safe_unicode, we mock it now, but should remove it
 safe_unicode = safe_str
 
 
 def __get_lem(extra_mapping=None):
     """
     Get language extension map based on what's inside pygments lexers
     """
     d = collections.defaultdict(lambda: [])
 
     def __clean(s):
         s = s.lstrip('*')
         s = s.lstrip('.')
 
         if s.find('[') != -1:
             exts = []
             start, stop = s.find('['), s.find(']')
 
             for suffix in s[start + 1:stop]:
                 exts.append(s[:s.find('[')] + suffix)
             return [e.lower() for e in exts]
         else:
             return [s.lower()]
 
     for lx, t in sorted(pygments.lexers.LEXERS.items()):
         m = list(map(__clean, t[-2]))
         if m:
             m = reduce(lambda x, y: x + y, m)
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
 
     data = dict(d)
 
     extra_mapping = extra_mapping or {}
     if extra_mapping:
         for k, v in extra_mapping.items():
             if k not in data:
                 # register new mapping2lexer
                 data[k] = [v]
 
     return data
 
 
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" accordingly to given mode
 
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
 
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     if mode == 0:
         line = line.replace('\r\n', '\n')
         line = line.replace('\r', '\n')
     elif mode == 1:
         line = line.replace('\r\n', '\r')
         line = line.replace('\n', '\r')
     elif mode == 2:
         line = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', line)
     return line
 
 
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def remove_suffix(s, suffix):
     if s.endswith(suffix):
         s = s[:-1 * len(suffix)]
     return s
 
 
 def remove_prefix(s, prefix):
     if s.startswith(prefix):
         s = s[len(prefix):]
     return s
 
 
 def find_calling_context(ignore_modules=None):
     """
     Look through the calling stack and return the frame which called
     this function and is part of core module ( ie. rhodecode.* )
 
     :param ignore_modules: list of modules to ignore eg. ['rhodecode.lib']
 
     usage::
         from rhodecode.lib.utils2 import find_calling_context
 
         calling_context = find_calling_context(ignore_modules=[
             'rhodecode.lib.caching_query',
             'rhodecode.model.settings',
         ])
 
         if calling_context:
             cc_str = 'call context %s:%s' % (
                 calling_context.f_code.co_filename,
                 calling_context.f_lineno,
             )
             print(cc_str)
     """
 
     ignore_modules = ignore_modules or []
 
     f = sys._getframe(2)
     while f.f_back is not None:
         name = f.f_globals.get('__name__')
         if name and name.startswith(__name__.split('.')[0]):
             if name not in ignore_modules:
                 return f
         f = f.f_back
     return None
 
 
 def ping_connection(connection, branch):
     if branch:
         # "branch" refers to a sub-connection of a connection,
         # we don't want to bother pinging on these.
         return
 
     # turn off "close with result".  This flag is only used with
     # "connectionless" execution, otherwise will be False in any case
     save_should_close_with_result = connection.should_close_with_result
     connection.should_close_with_result = False
 
     try:
         # run a SELECT 1.   use a core select() so that
         # the SELECT of a scalar value without a table is
         # appropriately formatted for the backend
         connection.scalar(sqlalchemy.sql.select([1]))
     except sqlalchemy.exc.DBAPIError as err:
         # catch SQLAlchemy's DBAPIError, which is a wrapper
         # for the DBAPI's exception.  It includes a .connection_invalidated
         # attribute which specifies if this connection is a "disconnect"
         # condition, which is based on inspection of the original exception
         # by the dialect in use.
         if err.connection_invalidated:
             # run the same SELECT again - the connection will re-validate
             # itself and establish a new connection.  The disconnect detection
             # here also causes the whole connection pool to be invalidated
             # so that all stale connections are discarded.
             connection.scalar(sqlalchemy.sql.select([1]))
         else:
             raise
     finally:
         # restore "close with result"
         connection.should_close_with_result = save_should_close_with_result
 
 
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """Custom engine_from_config functions."""
     log = logging.getLogger('sqlalchemy.engine')
     use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None))
     debug = asbool(configuration.pop('sqlalchemy.db1.debug_query', None))
 
     engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs)
 
     def color_sql(sql):
         color_seq = '\033[1;33m'  # This is yellow: code 33
         normal = '\x1b[0m'
         return ''.join([color_seq, sql, normal])
 
     if use_ping_connection:
         log.debug('Adding ping_connection on the engine config.')
         sqlalchemy.event.listen(engine, "engine_connect", ping_connection)
 
     if debug:
         # attach events only for debug configuration
         def before_cursor_execute(conn, cursor, statement,
                                   parameters, context, executemany):
             setattr(conn, 'query_start_time', time.time())
             log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
             calling_context = find_calling_context(ignore_modules=[
                 'rhodecode.lib.caching_query',
                 'rhodecode.model.settings',
             ])
             if calling_context:
                 log.info(color_sql('call context %s:%s' % (
                     calling_context.f_code.co_filename,
                     calling_context.f_lineno,
                 )))
 
         def after_cursor_execute(conn, cursor, statement,
                                  parameters, context, executemany):
             delattr(conn, 'query_start_time')
 
         sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute)
         sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute)
 
     return engine
 
 
 def get_encryption_key(config):
     secret = config.get('rhodecode.encrypted_values.secret')
     default = config['beaker.session.secret']
     return secret or default
 
 
 def age(prevdate, now=None, show_short_version=False, show_suffix=True, short_format=False):
     """
     Turns a datetime into an age string.
     If show_short_version is True, this generates a shorter string with
     an approximate age; ex. '1 day ago', rather than '1 day and 23 hours ago'.
 
     * IMPORTANT*
     Code of this function is written in special way so it's easier to
     backport it to javascript. If you mean to update it, please also update
     `jquery.timeago-extension.js` file
 
     :param prevdate: datetime object
     :param now: get current time, if not define we use
         `datetime.datetime.now()`
     :param show_short_version: if it should approximate the date and
         return a shorter string
     :param show_suffix:
     :param short_format: show short format, eg 2D instead of 2 days
     :rtype: unicode
     :returns: unicode words describing age
     """
 
     def _get_relative_delta(now, prevdate):
         base = dateutil.relativedelta.relativedelta(now, prevdate)
         return {
             'year': base.years,
             'month': base.months,
             'day': base.days,
             'hour': base.hours,
             'minute': base.minutes,
             'second': base.seconds,
         }
 
     def _is_leap_year(year):
         return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
 
     def get_month(prevdate):
         return prevdate.month
 
     def get_year(prevdate):
         return prevdate.year
 
     now = now or datetime.datetime.now()
     order = ['year', 'month', 'day', 'hour', 'minute', 'second']
     deltas = {}
     future = False
 
     if prevdate > now:
         now_old = now
         now = prevdate
         prevdate = now_old
         future = True
     if future:
         prevdate = prevdate.replace(microsecond=0)
     # Get date parts deltas
     for part in order:
         rel_delta = _get_relative_delta(now, prevdate)
         deltas[part] = rel_delta[part]
 
     # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
     # not 1 hour, -59 minutes and -59 seconds)
     offsets = [[5, 60], [4, 60], [3, 24]]
     for element in offsets:  # seconds, minutes, hours
         num = element[0]
         length = element[1]
 
         part = order[num]
         carry_part = order[num - 1]
 
         if deltas[part] < 0:
             deltas[part] += length
             deltas[carry_part] -= 1
 
     # Same thing for days except that the increment depends on the (variable)
     # number of days in the month
     month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
     if deltas['day'] < 0:
         if get_month(prevdate) == 2 and _is_leap_year(get_year(prevdate)):
             deltas['day'] += 29
         else:
             deltas['day'] += month_lengths[get_month(prevdate) - 1]
 
         deltas['month'] -= 1
 
     if deltas['month'] < 0:
         deltas['month'] += 12
         deltas['year'] -= 1
 
     # Format the result
     if short_format:
         fmt_funcs = {
-            'year': lambda d: u'%dy' % d,
-            'month': lambda d: u'%dm' % d,
-            'day': lambda d: u'%dd' % d,
-            'hour': lambda d: u'%dh' % d,
-            'minute': lambda d: u'%dmin' % d,
-            'second': lambda d: u'%dsec' % d,
+            'year': lambda d: '%dy' % d,
+            'month': lambda d: '%dm' % d,
+            'day': lambda d: '%dd' % d,
+            'hour': lambda d: '%dh' % d,
+            'minute': lambda d: '%dmin' % d,
+            'second': lambda d: '%dsec' % d,
         }
     else:
         fmt_funcs = {
-            'year': lambda d: _pluralize(u'${num} year', u'${num} years', d, mapping={'num': d}).interpolate(),
-            'month': lambda d: _pluralize(u'${num} month', u'${num} months', d, mapping={'num': d}).interpolate(),
-            'day': lambda d: _pluralize(u'${num} day', u'${num} days', d, mapping={'num': d}).interpolate(),
-            'hour': lambda d: _pluralize(u'${num} hour', u'${num} hours', d, mapping={'num': d}).interpolate(),
-            'minute': lambda d: _pluralize(u'${num} minute', u'${num} minutes', d, mapping={'num': d}).interpolate(),
-            'second': lambda d: _pluralize(u'${num} second', u'${num} seconds', d, mapping={'num': d}).interpolate(),
+            'year': lambda d: _pluralize('${num} year', '${num} years', d, mapping={'num': d}).interpolate(),
+            'month': lambda d: _pluralize('${num} month', '${num} months', d, mapping={'num': d}).interpolate(),
+            'day': lambda d: _pluralize('${num} day', '${num} days', d, mapping={'num': d}).interpolate(),
+            'hour': lambda d: _pluralize('${num} hour', '${num} hours', d, mapping={'num': d}).interpolate(),
+            'minute': lambda d: _pluralize('${num} minute', '${num} minutes', d, mapping={'num': d}).interpolate(),
+            'second': lambda d: _pluralize('${num} second', '${num} seconds', d, mapping={'num': d}).interpolate(),
         }
 
     i = 0
     for part in order:
         value = deltas[part]
         if value != 0:
 
             if i < 5:
                 sub_part = order[i + 1]
                 sub_value = deltas[sub_part]
             else:
                 sub_value = 0
 
             if sub_value == 0 or show_short_version:
                 _val = fmt_funcs[part](value)
                 if future:
                     if show_suffix:
-                        return _(u'in ${ago}', mapping={'ago': _val})
+                        return _('in ${ago}', mapping={'ago': _val})
                     else:
                         return _(_val)
 
                 else:
                     if show_suffix:
-                        return _(u'${ago} ago', mapping={'ago': _val})
+                        return _('${ago} ago', mapping={'ago': _val})
                     else:
                         return _(_val)
 
             val = fmt_funcs[part](value)
             val_detail = fmt_funcs[sub_part](sub_value)
             mapping = {'val': val, 'detail': val_detail}
 
             if short_format:
-                datetime_tmpl = _(u'${val}, ${detail}', mapping=mapping)
+                datetime_tmpl = _('${val}, ${detail}', mapping=mapping)
                 if show_suffix:
-                    datetime_tmpl = _(u'${val}, ${detail} ago', mapping=mapping)
+                    datetime_tmpl = _('${val}, ${detail} ago', mapping=mapping)
                     if future:
-                        datetime_tmpl = _(u'in ${val}, ${detail}', mapping=mapping)
+                        datetime_tmpl = _('in ${val}, ${detail}', mapping=mapping)
             else:
-                datetime_tmpl = _(u'${val} and ${detail}', mapping=mapping)
+                datetime_tmpl = _('${val} and ${detail}', mapping=mapping)
                 if show_suffix:
-                    datetime_tmpl = _(u'${val} and ${detail} ago', mapping=mapping)
+                    datetime_tmpl = _('${val} and ${detail} ago', mapping=mapping)
                     if future:
-                        datetime_tmpl = _(u'in ${val} and ${detail}', mapping=mapping)
+                        datetime_tmpl = _('in ${val} and ${detail}', mapping=mapping)
 
             return datetime_tmpl
         i += 1
-    return _(u'just now')
+    return _('just now')
 
 
 def age_from_seconds(seconds):
     seconds = safe_int(seconds) or 0
     prevdate = time_to_datetime(time.time() + seconds)
     return age(prevdate, show_suffix=False, show_short_version=True)
 
 
 def cleaned_uri(uri):
     """
     Quotes '[' and ']' from uri if there is only one of them.
         according to RFC3986 we cannot use such chars in uri
     :param uri:
     :return: uri without this chars
     """
     return urllib.parse.quote(uri, safe='@$:/')
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
 
     :param uri:
     """
     import urlobject
     if isinstance(uri, rhodecode.lib.encrypt.InvalidDecryptedValue):
         return 'InvalidDecryptionKey'
 
     url_obj = urlobject.URLObject(cleaned_uri(uri))
     url_obj = url_obj.without_password().without_username()
 
     return url_obj
 
 
 def get_host_info(request):
     """
     Generate host info, to obtain full url e.g https://server.com
     use this
     `{scheme}://{netloc}`
     """
     if not request:
         return {}
 
     qualified_home_url = request.route_url('home')
     parsed_url = urlobject.URLObject(qualified_home_url)
     decoded_path = safe_unicode(urllib.parse.unquote(parsed_url.path.rstrip('/')))
 
     return {
         'scheme': parsed_url.scheme,
         'netloc': parsed_url.netloc+decoded_path,
         'hostname': parsed_url.hostname,
     }
 
 
 def get_clone_url(request, uri_tmpl, repo_name, repo_id, repo_type, **override):
     qualified_home_url = request.route_url('home')
     parsed_url = urlobject.URLObject(qualified_home_url)
     decoded_path = safe_unicode(urllib.parse.unquote(parsed_url.path.rstrip('/')))
 
     args = {
         'scheme': parsed_url.scheme,
         'user': '',
         'sys_user': getpass.getuser(),
         # path if we use proxy-prefix
         'netloc': parsed_url.netloc+decoded_path,
         'hostname': parsed_url.hostname,
         'prefix': decoded_path,
         'repo': repo_name,
         'repoid': str(repo_id),
         'repo_type': repo_type
     }
     args.update(override)
     args['user'] = urllib.parse.quote(safe_str(args['user']))
 
     for k, v in args.items():
         uri_tmpl = uri_tmpl.replace('{%s}' % k, v)
 
     # special case for SVN clone url
     if repo_type == 'svn':
         uri_tmpl = uri_tmpl.replace('ssh://', 'svn+ssh://')
 
     # remove leading @ sign if it's present. Case of empty user
     url_obj = urlobject.URLObject(uri_tmpl)
     url = url_obj.with_netloc(url_obj.netloc.lstrip('@'))
 
     return safe_unicode(url)
 
 
 def get_commit_safe(repo, commit_id=None, commit_idx=None, pre_load=None,
                     maybe_unreachable=False, reference_obj=None):
     """
     Safe version of get_commit if this commit doesn't exists for a
     repository it returns a Dummy one instead
 
     :param repo: repository instance
     :param commit_id: commit id as str
     :param commit_idx: numeric commit index
     :param pre_load: optional list of commit attributes to load
     :param maybe_unreachable: translate unreachable commits on git repos
     :param reference_obj: explicitly search via a reference obj in git. E.g "branch:123" would mean branch "123"
     """
     # TODO(skreft): remove these circular imports
     from rhodecode.lib.vcs.backends.base import BaseRepository, EmptyCommit
     from rhodecode.lib.vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
 
     try:
         commit = repo.get_commit(
             commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load,
             maybe_unreachable=maybe_unreachable, reference_obj=reference_obj)
     except (RepositoryError, LookupError):
         commit = EmptyCommit()
     return commit
 
 
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 
 
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 
 
 def time_to_utcdatetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.utcfromtimestamp(tm)
 
 
 MENTIONS_REGEX = re.compile(
     # ^@ or @ without any special chars in front
     r'(?:^@|[^a-zA-Z0-9\-\_\.]@)'
     # main body starts with letter, then can be . - _
     r'([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)',
     re.VERBOSE | re.MULTILINE)
 
 
 def extract_mentioned_users(s):
     """
     Returns unique usernames from given string s that have @mention
 
     :param s: string to get mentions
     """
     usrs = set()
     for username in MENTIONS_REGEX.findall(s):
         usrs.add(username)
 
     return sorted(list(usrs), key=lambda k: k.lower())
 
 
 class AttributeDictBase(dict):
     def __getstate__(self):
         odict = self.__dict__  # get attribute dictionary
         return odict
 
     def __setstate__(self, dict):
         self.__dict__ = dict
 
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
 class StrictAttributeDict(AttributeDictBase):
     """
     Strict Version of Attribute dict which raises an Attribute error when
     requested attribute is not set
     """
     def __getattr__(self, attr):
         try:
             return self[attr]
         except KeyError:
             raise AttributeError('%s object has no attribute %s' % (
                 self.__class__, attr))
 
 
 class AttributeDict(AttributeDictBase):
     def __getattr__(self, attr):
         return self.get(attr, None)
 
 
 def fix_PATH(os_=None):
     """
     Get current active python path, and append it to PATH variable to fix
     issues of subprocess calls and different python versions
     """
     if os_ is None:
         import os
     else:
         os = os_
 
     cur_path = os.path.split(sys.executable)[0]
     if not os.environ['PATH'].startswith(cur_path):
         os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
 
 
 def obfuscate_url_pw(engine):
     _url = engine or ''
     try:
         _url = sqlalchemy.engine.url.make_url(engine)
         if _url.password:
             _url.password = 'XXXXX'
     except Exception:
         pass
     return str(_url)
 
 
 def get_server_url(environ):
     req = webob.Request(environ)
     return req.host_url + req.script_name
 
 
 def unique_id(hexlen=32):
     alphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz"
     return suuid(truncate_to=hexlen, alphabet=alphabet)
 
 
 def suuid(url=None, truncate_to=22, alphabet=None):
     """
     Generate and return a short URL safe UUID.
 
     If the url parameter is provided, set the namespace to the provided
     URL and generate a UUID.
 
     :param url to get the uuid for
     :truncate_to: truncate the basic 22 UUID to shorter version
 
     The IDs won't be universally unique any longer, but the probability of
     a collision will still be very low.
     """
     # Define our alphabet.
     _ALPHABET = alphabet or "23456789ABCDEFGHJKLMNPQRSTUVWXYZ"
 
     # If no URL is given, generate a random UUID.
     if url is None:
         unique_id = uuid.uuid4().int
     else:
         unique_id = uuid.uuid3(uuid.NAMESPACE_URL, url).int
 
     alphabet_length = len(_ALPHABET)
     output = []
     while unique_id > 0:
         digit = unique_id % alphabet_length
         output.append(_ALPHABET[digit])
         unique_id = int(unique_id / alphabet_length)
     return "".join(output)[:truncate_to]
 
 
 def get_current_rhodecode_user(request=None):
     """
     Gets rhodecode user from request
     """
     pyramid_request = request or pyramid.threadlocal.get_current_request()
 
     # web case
     if pyramid_request and hasattr(pyramid_request, 'user'):
         return pyramid_request.user
 
     # api case
     if pyramid_request and hasattr(pyramid_request, 'rpc_user'):
         return pyramid_request.rpc_user
 
     return None
 
 
 def action_logger_generic(action, namespace=''):
     """
     A generic logger for actions useful to the system overview, tries to find
     an acting user for the context of the call otherwise reports unknown user
 
     :param action: logging message eg 'comment 5 deleted'
     :param type: string
 
     :param namespace: namespace of the logging message eg. 'repo.comments'
     :param type: string
 
     """
 
     logger_name = 'rhodecode.actions'
 
     if namespace:
         logger_name += '.' + namespace
 
     log = logging.getLogger(logger_name)
 
     # get a user if we can
     user = get_current_rhodecode_user()
 
     logfunc = log.info
 
     if not user:
         user = '<unknown user>'
         logfunc = log.warning
 
     logfunc('Logging action by {}: {}'.format(user, action))
 
 
 def escape_split(text, sep=',', maxsplit=-1):
     r"""
     Allows for escaping of the separator: e.g. arg='foo\, bar'
 
     It should be noted that the way bash et. al. do command line parsing, those
     single quotes are required.
     """
     escaped_sep = r'\%s' % sep
 
     if escaped_sep not in text:
         return text.split(sep, maxsplit)
 
     before, _mid, after = text.partition(escaped_sep)
     startlist = before.split(sep, maxsplit)  # a regular split is fine here
     unfinished = startlist[-1]
     startlist = startlist[:-1]
 
     # recurse because there may be more escaped separators
     endlist = escape_split(after, sep, maxsplit)
 
     # finish building the escaped value. we use endlist[0] becaue the first
     # part of the string sent in recursion is the rest of the escaped value.
     unfinished += sep + endlist[0]
 
     return startlist + [unfinished] + endlist[1:]  # put together all the parts
 
 
 class OptionalAttr(object):
     """
     Special Optional Option that defines other attribute. Example::
 
         def test(apiuser, userid=Optional(OAttr('apiuser')):
             user = Optional.extract(userid)
             # calls
 
     """
 
     def __init__(self, attr_name):
         self.attr_name = attr_name
 
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
 
     def __call__(self):
         return self
 
 
 # alias
 OAttr = OptionalAttr
 
 
 class Optional(object):
     """
     Defines an optional parameter::
 
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
 
     is equivalent of::
 
         param = Optional.extract(param)
 
     """
 
     def __init__(self, type_):
         self.type_ = type_
 
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
 
     def __call__(self):
         return self.getval()
 
     def getval(self):
         """
         returns value from this Optional instance
         """
         if isinstance(self.type_, OAttr):
             # use params name
             return self.type_.attr_name
         return self.type_
 
     @classmethod
     def extract(cls, val):
         """
         Extracts value from Optional() instance
 
         :param val:
         :return: original value if it's not Optional instance else
             value of instance
         """
         if isinstance(val, cls):
             return val.getval()
         return val
 
 
 def glob2re(pat):
     """
     Translate a shell PATTERN to a regular expression.
 
     There is no way to quote meta-characters.
     """
 
     i, n = 0, len(pat)
     res = ''
     while i < n:
         c = pat[i]
         i = i+1
         if c == '*':
             #res = res + '.*'
             res = res + '[^/]*'
         elif c == '?':
             #res = res + '.'
             res = res + '[^/]'
         elif c == '[':
             j = i
             if j < n and pat[j] == '!':
                 j = j+1
             if j < n and pat[j] == ']':
                 j = j+1
             while j < n and pat[j] != ']':
                 j = j+1
             if j >= n:
                 res = res + '\\['
             else:
                 stuff = pat[i:j].replace('\\','\\\\')
                 i = j+1
                 if stuff[0] == '!':
                     stuff = '^' + stuff[1:]
                 elif stuff[0] == '^':
                     stuff = '\\' + stuff
                 res = '%s[%s]' % (res, stuff)
         else:
             res = res + re.escape(c)
     return res + '\Z(?ms)'
 
 
 def parse_byte_string(size_str):
     match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE)
     if not match:
         raise ValueError('Given size:%s is invalid, please make sure '
                          'to use format of <num>(MB|KB)' % size_str)
 
     _parts = match.groups()
     num, type_ = _parts
     return int(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()]
 
 
 class CachedProperty(object):
     """
     Lazy Attributes. With option to invalidate the cache by running a method
 
     >>> class Foo(object):
     ...
     ...    @CachedProperty
     ...    def heavy_func(self):
     ...        return 'super-calculation'
     ...
     ... foo = Foo()
     ... foo.heavy_func() # first computation
     ... foo.heavy_func() # fetch from cache
     ... foo._invalidate_prop_cache('heavy_func')
 
     # at this point calling foo.heavy_func() will be re-computed
     """
 
     def __init__(self, func, func_name=None):
 
         if func_name is None:
             func_name = func.__name__
         self.data = (func, func_name)
         update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:
             return self
 
         func, func_name = self.data
         value = func(inst)
         inst.__dict__[func_name] = value
         if '_invalidate_prop_cache' not in inst.__dict__:
             inst.__dict__['_invalidate_prop_cache'] = partial(
                 self._invalidate_prop_cache, inst)
         return value
 
     def _invalidate_prop_cache(self, inst, name):
         inst.__dict__.pop(name, None)
 
 
 def retry(func=None, exception=Exception, n_tries=5, delay=5, backoff=1, logger=True):
     """
     Retry decorator with exponential backoff.
 
     Parameters
     ----------
     func : typing.Callable, optional
         Callable on which the decorator is applied, by default None
     exception : Exception or tuple of Exceptions, optional
         Exception(s) that invoke retry, by default Exception
     n_tries : int, optional
         Number of tries before giving up, by default 5
     delay : int, optional
         Initial delay between retries in seconds, by default 5
     backoff : int, optional
         Backoff multiplier e.g. value of 2 will double the delay, by default 1
     logger : bool, optional
         Option to log or print, by default False
 
     Returns
     -------
     typing.Callable
         Decorated callable that calls itself when exception(s) occur.
 
     Examples
     --------
     >>> import random
     >>> @retry(exception=Exception, n_tries=3)
     ... def test_random(text):
     ...    x = random.random()
     ...    if x < 0.5:
     ...        raise Exception("Fail")
     ...    else:
     ...        print("Success: ", text)
     >>> test_random("It works!")
     """
 
     if func is None:
         return partial(
             retry,
             exception=exception,
             n_tries=n_tries,
             delay=delay,
             backoff=backoff,
             logger=logger,
         )
 
     @wraps(func)
     def wrapper(*args, **kwargs):
         _n_tries, n_delay = n_tries, delay
         log = logging.getLogger('rhodecode.retry')
 
         while _n_tries > 1:
             try:
                 return func(*args, **kwargs)
             except exception as e:
                 e_details = repr(e)
                 msg = "Exception on calling func {func}: {e}, " \
                       "Retrying in {n_delay} seconds..."\
                     .format(func=func, e=e_details, n_delay=n_delay)
                 if logger:
                     log.warning(msg)
                 else:
                     print(msg)
                 time.sleep(n_delay)
                 _n_tries -= 1
                 n_delay *= backoff
 
         return func(*args, **kwargs)
 
     return wrapper
 
 
 def user_agent_normalizer(user_agent_raw, safe=True):
     log = logging.getLogger('rhodecode.user_agent_normalizer')
     ua = (user_agent_raw or '').strip().lower()
     ua = ua.replace('"', '')
 
     try:
         if 'mercurial/proto-1.0' in ua:
             ua = ua.replace('mercurial/proto-1.0', '')
             ua = ua.replace('(', '').replace(')', '').strip()
             ua = ua.replace('mercurial ', 'mercurial/')
         elif ua.startswith('git'):
             parts = ua.split(' ')
             if parts:
                 ua = parts[0]
             ua = re.sub('\.windows\.\d', '', ua).strip()
 
         return ua
     except Exception:
         log.exception('Failed to parse scm user-agent')
         if not safe:
             raise
 
     return ua
 
 
 def get_available_port(min_port=40000, max_port=55555, use_range=False):
     hostname = ''
     for _ in range(min_port, max_port):
         pick_port = 0
         if use_range:
             pick_port = random.randint(min_port, max_port)
 
         with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
             try:
                 s.bind((hostname, pick_port))
                 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                 return s.getsockname()[1]
             except OSError:
                 continue
             except socket.error as e:
                 if e.args[0] in [errno.EADDRINUSE, errno.ECONNREFUSED]:
                     continue
                 raise

rhodecode/lib/vcs/backends/git/commit.py

@@ -1,494 +1,494 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT commit module
 """
 
 import re
+import io
 import stat
 import configparser
 from itertools import chain
-from io import StringIO
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
 from rhodecode.lib.vcs.nodes import (
     FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
     ChangedFileNodesGenerator, AddedFileNodesGenerator,
     RemovedFileNodesGenerator, LargeFileNode)
 
 
 class GitCommit(base.BaseCommit):
     """
     Represents state of the repository at single commit id.
     """
 
     _filter_pre_load = [
         # done through a more complex tree walk on parents
         "affected_files",
         # done through subprocess not remote call
         "children",
         # done through a more complex tree walk on parents
         "status",
         # mercurial specific property not supported here
         "_file_paths",
         # mercurial specific property not supported here
         'obsolete',
         # mercurial specific property not supported here
         'phase',
         # mercurial specific property not supported here
         'hidden'
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         self.repository = repository
         self._remote = repository._remote
         # TODO: johbo: Tweak of raw_id should not be necessary
         self.raw_id = safe_str(raw_id)
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self._stat_modes = {}  # stat info for paths
         self._paths = {}  # path processed with parse_tree
         self.nodes = {}
         self._submodules = None
 
     def _set_bulk_properties(self, pre_load):
 
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
         for attr, value in result.items():
             if attr in ["author", "message"]:
                 if value:
                     value = safe_unicode(value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr == "parents":
                 value = self._make_commits(value)
             elif attr == "branch":
                 value = self._set_branch(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def _commit(self):
         return self._remote[self.raw_id]
 
     @LazyProperty
     def _tree_id(self):
         return self._remote[self._commit['tree']]['id']
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def message(self):
         return safe_unicode(self._remote.message(self.id))
 
     @LazyProperty
     def committer(self):
         return safe_unicode(self._remote.author(self.id))
 
     @LazyProperty
     def author(self):
         return safe_unicode(self._remote.author(self.id))
 
     @LazyProperty
     def date(self):
         unix_ts, tz = self._remote.date(self.raw_id)
         return utcdate_fromtimestamp(unix_ts, tz)
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         return self.changed, self.added, self.removed
 
     @LazyProperty
     def tags(self):
         tags = [safe_unicode(name) for name,
                 commit_id in self.repository.tags.items()
                 if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def commit_branches(self):
         branches = []
         for name, commit_id in self.repository.branches.items():
             if commit_id == self.raw_id:
                 branches.append(name)
         return branches
 
     def _set_branch(self, branches):
         if branches:
             # actually commit can have multiple branches in git
             return safe_unicode(branches[0])
 
     @LazyProperty
     def branch(self):
         branches = self._remote.branch(self.raw_id)
         return self._set_branch(branches)
 
     def _get_tree_id_for_path(self, path):
         path = safe_str(path)
         if path in self._paths:
             return self._paths[path]
 
         tree_id = self._tree_id
 
         path = path.strip('/')
         if path == '':
             data = [tree_id, "tree"]
             self._paths[''] = data
             return data
 
         tree_id, tree_type, tree_mode = \
             self._remote.tree_and_type_for_path(self.raw_id, path)
         if tree_id is None:
             raise self.no_node_at_path(path)
 
         self._paths[path] = [tree_id, tree_type]
         self._stat_modes[path] = tree_mode
 
         if path not in self._paths:
             raise self.no_node_at_path(path)
 
         return self._paths[path]
 
     def _get_kind(self, path):
         tree_id, type_ = self._get_tree_id_for_path(path)
         if type_ == 'blob':
             return NodeKind.FILE
         elif type_ == 'tree':
             return NodeKind.DIR
         elif type_ == 'link':
             return NodeKind.SUBMODULE
         return None
 
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(
                 "File does not exist for commit %s at  '%s'" % (self.raw_id, path))
         return path
 
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parent_ids = self._remote.parents(self.id)
         return self._make_commits(parent_ids)
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
 
         children = self._remote.children(self.raw_id)
         return self._make_commits(children)
 
     def _make_commits(self, commit_ids):
         def commit_maker(_commit_id):
             return self.repository.get_commit(commit_id=commit_id)
 
         return [commit_maker(commit_id) for commit_id in commit_ids]
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given `path`.
         """
         path = safe_str(path)
         # ensure path is traversed
         self._get_tree_id_for_path(path)
         return self._stat_modes[path]
 
     def is_link(self, path):
         return stat.S_ISLNK(self.get_file_mode(path))
 
     def is_node_binary(self, path):
         tree_id, _ = self._get_tree_id_for_path(path)
         return self._remote.is_binary(tree_id)
 
     def get_file_content(self, path):
         """
         Returns content of the file at given `path`.
         """
         tree_id, _ = self._get_tree_id_for_path(path)
         return self._remote.blob_as_pretty_string(tree_id)
 
     def get_file_content_streamed(self, path):
         tree_id, _ = self._get_tree_id_for_path(path)
         stream_method = getattr(self._remote, 'stream:blob_as_pretty_string')
         return stream_method(tree_id)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given `path`.
         """
         tree_id, _ = self._get_tree_id_for_path(path)
         return self._remote.blob_raw_length(tree_id)
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `GitCommit` objects for
         which file at given `path` has been modified.
         """
 
         path = self._get_filectx(path)
         hist = self._remote.node_history(self.raw_id, path, limit)
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in hist]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
         """
 
         result = self._remote.node_annotate(self.raw_id, path)
 
         for ln_no, commit_id, content in result:
             yield (
                 ln_no, commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
                 content)
 
     def get_nodes(self, path):
 
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(
                 "Directory does not exist for commit %s at '%s'" % (self.raw_id, path))
         path = self._fix_path(path)
 
         tree_id, _ = self._get_tree_id_for_path(path)
 
         dirnodes = []
         filenodes = []
 
         # extracted tree ID gives us our files...
         bytes_path = safe_str(path)  # libgit operates on bytes
         for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
             if type_ == 'link':
                 url = self._get_submodule_url('/'.join((bytes_path, name)))
                 dirnodes.append(SubModuleNode(
                     name, url=url, commit=id_, alias=self.repository.alias))
                 continue
 
             if bytes_path != '':
                 obj_path = '/'.join((bytes_path, name))
             else:
                 obj_path = name
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat_
 
             if type_ == 'tree':
                 dirnodes.append(DirNode(obj_path, commit=self))
             elif type_ == 'blob':
                 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
             else:
                 raise CommitError(
                     "Requested object should be Tree or Blob, is %s", type_)
 
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
 
     def get_node(self, path, pre_load=None):
         path = self._fix_path(path)
         if path not in self.nodes:
             try:
                 tree_id, type_ = self._get_tree_id_for_path(path)
             except CommitError:
                 raise NodeDoesNotExistError(
                     "Cannot find one of parents' directories for a given "
                     "path: %s" % path)
 
             if type_ in ['link', 'commit']:
                 url = self._get_submodule_url(path)
                 node = SubModuleNode(path, url=url, commit=tree_id,
                                      alias=self.repository.alias)
             elif type_ == 'tree':
                 if path == '':
                     node = RootNode(commit=self)
                 else:
                     node = DirNode(path, commit=self)
             elif type_ == 'blob':
                 node = FileNode(path, commit=self, pre_load=pre_load)
                 self._stat_modes[path] = node.mode
             else:
                 raise self.no_node_at_path(path)
 
             # cache node
             self.nodes[path] = node
 
         return self.nodes[path]
 
     def get_largefile_node(self, path):
         tree_id, _ = self._get_tree_id_for_path(path)
         pointer_spec = self._remote.is_large_file(tree_id)
 
         if pointer_spec:
             # content of that file regular FileNode is the hash of largefile
             file_id = pointer_spec.get('oid_hash')
             if self._remote.in_largefiles_store(file_id):
                 lf_path = self._remote.store_path(file_id)
                 return LargeFileNode(lf_path, commit=self, org_path=path)
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given commit
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
 
     @LazyProperty
     def _changes_cache(self):
         added = set()
         modified = set()
         deleted = set()
         _r = self._remote
 
         parents = self.parents
         if not self.parents:
             parents = [base.EmptyCommit()]
         for parent in parents:
             if isinstance(parent, base.EmptyCommit):
                 oid = None
             else:
                 oid = parent.raw_id
             changes = _r.tree_changes(oid, self.raw_id)
             for (oldpath, newpath), (_, _), (_, _) in changes:
                 if newpath and oldpath:
                     modified.add(newpath)
                 elif newpath and not oldpath:
                     added.add(newpath)
                 elif not newpath and oldpath:
                     deleted.add(oldpath)
         return added, modified, deleted
 
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
 
         :param status: one of: *added*, *modified* or *deleted*
         """
         added, modified, deleted = self._changes_cache
         return sorted({
             'added': list(added),
             'modified': list(modified),
             'deleted': list(deleted)}[status]
         )
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         if not self.parents:
             return list(self._get_file_nodes())
         return AddedFileNodesGenerator(self.added_paths, self)
 
     @LazyProperty
     def added_paths(self):
         return [n for n in self._get_paths_for_status('added')]
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return ChangedFileNodesGenerator(self.changed_paths, self)
 
     @LazyProperty
     def changed_paths(self):
         return [n for n in self._get_paths_for_status('modified')]
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return RemovedFileNodesGenerator(self.removed_paths, self)
 
     @LazyProperty
     def removed_paths(self):
         return [n for n in self._get_paths_for_status('deleted')]
 
     def _get_submodule_url(self, submodule_path):
         git_modules_path = '.gitmodules'
 
         if self._submodules is None:
             self._submodules = {}
 
             try:
                 submodules_node = self.get_node(git_modules_path)
             except NodeDoesNotExistError:
                 return None
 
             # ConfigParser fails if there are whitespaces, also it needs an iterable
             # file like content
             def iter_content(_content):
                 for line in _content.splitlines():
                     yield line
 
             parser = configparser.RawConfigParser()
             parser.read_file(iter_content(submodules_node.content))
 
             for section in parser.sections():
                 path = parser.get(section, 'path')
                 url = parser.get(section, 'url')
                 if path and url:
                     self._submodules[path.strip('/')] = url
 
         return self._submodules.get(submodule_path.strip('/'))

rhodecode/lib/vcs/backends/hg/commit.py

@@ -1,401 +1,402 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 HG commit module
 """
 
 import os
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.utils import safe_str, safe_unicode
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
 from rhodecode.lib.vcs.exceptions import CommitError
 from rhodecode.lib.vcs.nodes import (
     AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode,
     NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode,
     LargeFileNode, LARGEFILE_PREFIX)
 from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
 
 
 class MercurialCommit(base.BaseCommit):
     """
     Represents state of the repository at the single commit.
     """
 
     _filter_pre_load = [
         # git specific property not supported here
         "_commit",
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         raw_id = safe_str(raw_id)
 
         self.repository = repository
         self._remote = repository._remote
 
         self.raw_id = raw_id
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self.nodes = {}
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
+
         for attr, value in result.items():
             if attr in ["author", "branch", "message"]:
                 value = safe_unicode(value)
             elif attr == "affected_files":
                 value = map(safe_unicode, value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr in ["children", "parents"]:
                 value = self._make_commits(value)
             elif attr in ["phase"]:
                 value = self._get_phase_text(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def tags(self):
         tags = [name for name, commit_id in self.repository.tags.items()
                 if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def branch(self):
         return safe_unicode(self._remote.ctx_branch(self.raw_id))
 
     @LazyProperty
     def bookmarks(self):
         bookmarks = [
             name for name, commit_id in self.repository.bookmarks.items()
             if commit_id == self.raw_id]
         return bookmarks
 
     @LazyProperty
     def message(self):
         return safe_unicode(self._remote.ctx_description(self.raw_id))
 
     @LazyProperty
     def committer(self):
         return safe_unicode(self.author)
 
     @LazyProperty
     def author(self):
         return safe_unicode(self._remote.ctx_user(self.raw_id))
 
     @LazyProperty
     def date(self):
         return utcdate_fromtimestamp(*self._remote.ctx_date(self.raw_id))
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         return self._remote.ctx_status(self.raw_id)
 
     @LazyProperty
     def _file_paths(self):
         return self._remote.ctx_list(self.raw_id)
 
     @LazyProperty
     def _dir_paths(self):
         p = list(set(get_dirs_for_path(*self._file_paths)))
         p.insert(0, '')
         return p
 
     @LazyProperty
     def _paths(self):
         return self._dir_paths + self._file_paths
 
     @LazyProperty
     def id(self):
         if self.last:
             return u'tip'
         return self.short_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def _make_commits(self, commit_ids, pre_load=None):
         return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
                 for commit_id in commit_ids]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parents = self._remote.ctx_parents(self.raw_id)
         return self._make_commits(parents)
 
     def _get_phase_text(self, phase_id):
         return {
                    0: 'public',
                    1: 'draft',
                    2: 'secret',
                }.get(phase_id) or ''
 
     @LazyProperty
     def phase(self):
         phase_id = self._remote.ctx_phase(self.raw_id)
         phase_text = self._get_phase_text(phase_id)
 
         return safe_unicode(phase_text)
 
     @LazyProperty
     def obsolete(self):
         obsolete = self._remote.ctx_obsolete(self.raw_id)
         return obsolete
 
     @LazyProperty
     def hidden(self):
         hidden = self._remote.ctx_hidden(self.raw_id)
         return hidden
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         children = self._remote.ctx_children(self.raw_id)
         return self._make_commits(children)
 
     def _fix_path(self, path):
         """
         Mercurial keeps filenodes as str so we need to encode from unicode
         to str.
         """
         return safe_str(super(MercurialCommit, self)._fix_path(path))
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         if path in self._file_paths:
             return NodeKind.FILE
         elif path in self._dir_paths:
             return NodeKind.DIR
         else:
             raise CommitError(
                 "Node does not exist at the given path '%s'" % (path, ))
 
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(
                 "File does not exist for idx %s at '%s'" % (self.raw_id, path))
         return path
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         path = self._get_filectx(path)
         if 'x' in self._remote.fctx_flags(self.raw_id, path):
             return base.FILEMODE_EXECUTABLE
         else:
             return base.FILEMODE_DEFAULT
 
     def is_link(self, path):
         path = self._get_filectx(path)
         return 'l' in self._remote.fctx_flags(self.raw_id, path)
 
     def is_node_binary(self, path):
         path = self._get_filectx(path)
         return self._remote.is_binary(self.raw_id, path)
 
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         path = self._get_filectx(path)
         return self._remote.fctx_node_data(self.raw_id, path)
 
     def get_file_content_streamed(self, path):
         path = self._get_filectx(path)
         stream_method = getattr(self._remote, 'stream:fctx_node_data')
         return stream_method(self.raw_id, path)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         path = self._get_filectx(path)
         return self._remote.fctx_size(self.raw_id, path)
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `MercurialCommit` objects
         for which file at given ``path`` has been modified.
         """
         path = self._get_filectx(path)
         hist = self._remote.node_history(self.raw_id, path, limit)
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in hist]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
         """
         result = self._remote.fctx_annotate(self.raw_id, path)
 
         for ln_no, commit_id, content in result:
             yield (
                 ln_no, commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
                 content)
 
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of commit at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, CommitError would be raised.
         """
 
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(
                 "Directory does not exist for idx %s at '%s'" % (self.raw_id, path))
         path = self._fix_path(path)
 
         filenodes = [
             FileNode(f, commit=self) for f in self._file_paths
             if os.path.dirname(f) == path]
         # TODO: johbo: Check if this can be done in a more obvious way
         dirs = path == '' and '' or [
             d for d in self._dir_paths
             if d and vcspath.dirname(d) == path]
         dirnodes = [
             DirNode(d, commit=self) for d in dirs
             if os.path.dirname(d) == path]
 
         alias = self.repository.alias
         for k, vals in self._submodules.items():
             if vcspath.dirname(k) == path:
                 loc = vals[0]
                 commit = vals[1]
                 dirnodes.append(SubModuleNode(k, url=loc, commit=commit, alias=alias))
 
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
 
         return nodes
 
     def get_node(self, path, pre_load=None):
         """
         Returns `Node` object from the given `path`. If there is no node at
         the given `path`, `NodeDoesNotExistError` would be raised.
         """
         path = self._fix_path(path)
 
         if path not in self.nodes:
             if path in self._file_paths:
                 node = FileNode(path, commit=self, pre_load=pre_load)
             elif path in self._dir_paths:
                 if path == '':
                     node = RootNode(commit=self)
                 else:
                     node = DirNode(path, commit=self)
             else:
                 raise self.no_node_at_path(path)
 
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     def get_largefile_node(self, path):
         pointer_spec = self._remote.is_large_file(self.raw_id, path)
         if pointer_spec:
             # content of that file regular FileNode is the hash of largefile
             file_id = self.get_file_content(path).strip()
 
             if self._remote.in_largefiles_store(file_id):
                 lf_path = self._remote.store_path(file_id)
                 return LargeFileNode(lf_path, commit=self, org_path=path)
             elif self._remote.in_user_cache(file_id):
                 lf_path = self._remote.store_path(file_id)
                 self._remote.link(file_id, path)
                 return LargeFileNode(lf_path, commit=self, org_path=path)
 
     @LazyProperty
     def _submodules(self):
         """
         Returns a dictionary with submodule information from substate file
         of hg repository.
         """
         return self._remote.ctx_substate(self.raw_id)
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given commit
         """
         return self._remote.ctx_files(self.raw_id)
 
     @property
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         return AddedFileNodesGenerator(self.added_paths, self)
 
     @LazyProperty
     def added_paths(self):
         return [n for n in self.status[1]]
 
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         return ChangedFileNodesGenerator(self.changed_paths, self)
 
     @LazyProperty
     def changed_paths(self):
         return [n for n in self.status[0]]
 
     @property
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         return RemovedFileNodesGenerator(self.removed_paths, self)
 
     @LazyProperty
     def removed_paths(self):
         return [n for n in self.status[2]]

rhodecode/model/permission.py

@@ -1,600 +1,600 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 permissions model for RhodeCode
 """
 import collections
 import logging
 import traceback
 
 from sqlalchemy.exc import DatabaseError
 
 from rhodecode import events
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     User, Permission, UserToPerm, UserRepoToPerm, UserRepoGroupToPerm,
     UserUserGroupToPerm, UserGroup, UserGroupToPerm, UserToRepoBranchPermission)
 from rhodecode.lib.utils2 import str2bool, safe_int
 
 log = logging.getLogger(__name__)
 
 
 class PermissionModel(BaseModel):
     """
     Permissions model for RhodeCode
     """
     FORKING_DISABLED = 'hg.fork.none'
     FORKING_ENABLED = 'hg.fork.repository'
 
     cls = Permission
     global_perms = {
         'default_repo_create': None,
         # special case for create repos on write access to group
         'default_repo_create_on_write': None,
         'default_repo_group_create': None,
         'default_user_group_create': None,
         'default_fork_create': None,
         'default_inherit_default_permissions': None,
         'default_register': None,
         'default_password_reset': None,
         'default_extern_activate': None,
 
         # object permissions below
         'default_repo_perm': None,
         'default_group_perm': None,
         'default_user_group_perm': None,
 
         # branch
         'default_branch_perm': None,
     }
 
     def set_global_permission_choices(self, c_obj, gettext_translator):
         _ = gettext_translator
 
         c_obj.repo_perms_choices = [
             ('repository.none', _('None'),),
             ('repository.read', _('Read'),),
             ('repository.write', _('Write'),),
             ('repository.admin', _('Admin'),)]
 
         c_obj.group_perms_choices = [
             ('group.none', _('None'),),
             ('group.read', _('Read'),),
             ('group.write', _('Write'),),
             ('group.admin', _('Admin'),)]
 
         c_obj.user_group_perms_choices = [
             ('usergroup.none', _('None'),),
             ('usergroup.read', _('Read'),),
             ('usergroup.write', _('Write'),),
             ('usergroup.admin', _('Admin'),)]
 
         c_obj.branch_perms_choices = [
             ('branch.none', _('Protected/No Access'),),
             ('branch.merge', _('Web merge'),),
             ('branch.push', _('Push'),),
             ('branch.push_force', _('Force Push'),)]
 
         c_obj.register_choices = [
             ('hg.register.none', _('Disabled')),
             ('hg.register.manual_activate', _('Allowed with manual account activation')),
-            ('hg.register.auto_activate', _('Allowed with automatic account activation')),]
+            ('hg.register.auto_activate', _('Allowed with automatic account activation'))]
 
         c_obj.password_reset_choices = [
             ('hg.password_reset.enabled', _('Allow password recovery')),
             ('hg.password_reset.hidden', _('Hide password recovery link')),
-            ('hg.password_reset.disabled', _('Disable password recovery')),]
+            ('hg.password_reset.disabled', _('Disable password recovery'))]
 
         c_obj.extern_activate_choices = [
             ('hg.extern_activate.manual', _('Manual activation of external account')),
-            ('hg.extern_activate.auto', _('Automatic activation of external account')),]
+            ('hg.extern_activate.auto', _('Automatic activation of external account'))]
 
         c_obj.repo_create_choices = [
             ('hg.create.none', _('Disabled')),
             ('hg.create.repository', _('Enabled'))]
 
         c_obj.repo_create_on_write_choices = [
             ('hg.create.write_on_repogroup.false', _('Disabled')),
             ('hg.create.write_on_repogroup.true', _('Enabled'))]
 
         c_obj.user_group_create_choices = [
             ('hg.usergroup.create.false', _('Disabled')),
             ('hg.usergroup.create.true', _('Enabled'))]
 
         c_obj.repo_group_create_choices = [
             ('hg.repogroup.create.false', _('Disabled')),
             ('hg.repogroup.create.true', _('Enabled'))]
 
         c_obj.fork_choices = [
             (self.FORKING_DISABLED, _('Disabled')),
             (self.FORKING_ENABLED, _('Enabled'))]
 
         c_obj.inherit_default_permission_choices = [
             ('hg.inherit_default_perms.false', _('Disabled')),
             ('hg.inherit_default_perms.true', _('Enabled'))]
 
     def get_default_perms(self, object_perms, suffix):
         defaults = {}
         for perm in object_perms:
             # perms
             if perm.permission.permission_name.startswith('repository.'):
                 defaults['default_repo_perm' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('group.'):
                 defaults['default_group_perm' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('usergroup.'):
                 defaults['default_user_group_perm' + suffix] = perm.permission.permission_name
 
             # branch
             if perm.permission.permission_name.startswith('branch.'):
                 defaults['default_branch_perm' + suffix] = perm.permission.permission_name
 
             # creation of objects
             if perm.permission.permission_name.startswith('hg.create.write_on_repogroup'):
                 defaults['default_repo_create_on_write' + suffix] = perm.permission.permission_name
 
             elif perm.permission.permission_name.startswith('hg.create.'):
                 defaults['default_repo_create' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.fork.'):
                 defaults['default_fork_create' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.inherit_default_perms.'):
                 defaults['default_inherit_default_permissions' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.repogroup.'):
                 defaults['default_repo_group_create' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.usergroup.'):
                 defaults['default_user_group_create' + suffix] = perm.permission.permission_name
 
             # registration and external account activation
             if perm.permission.permission_name.startswith('hg.register.'):
                 defaults['default_register' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.password_reset.'):
                 defaults['default_password_reset' + suffix] = perm.permission.permission_name
 
             if perm.permission.permission_name.startswith('hg.extern_activate.'):
                 defaults['default_extern_activate' + suffix] = perm.permission.permission_name
 
         return defaults
 
     def _make_new_user_perm(self, user, perm_name):
         log.debug('Creating new user permission:%s', perm_name)
         new = UserToPerm()
         new.user = user
         new.permission = Permission.get_by_key(perm_name)
         return new
 
     def _make_new_user_group_perm(self, user_group, perm_name):
         log.debug('Creating new user group permission:%s', perm_name)
         new = UserGroupToPerm()
         new.users_group = user_group
         new.permission = Permission.get_by_key(perm_name)
         return new
 
     def _keep_perm(self, perm_name, keep_fields):
         def get_pat(field_name):
             return {
                 # global perms
                 'default_repo_create': 'hg.create.',
                 # special case for create repos on write access to group
                 'default_repo_create_on_write': 'hg.create.write_on_repogroup.',
                 'default_repo_group_create': 'hg.repogroup.create.',
                 'default_user_group_create': 'hg.usergroup.create.',
                 'default_fork_create': 'hg.fork.',
                 'default_inherit_default_permissions': 'hg.inherit_default_perms.',
 
                 # application perms
                 'default_register': 'hg.register.',
                 'default_password_reset': 'hg.password_reset.',
                 'default_extern_activate': 'hg.extern_activate.',
 
                 # object permissions below
                 'default_repo_perm': 'repository.',
                 'default_group_perm': 'group.',
                 'default_user_group_perm': 'usergroup.',
                 # branch
                 'default_branch_perm': 'branch.',
 
             }[field_name]
         for field in keep_fields:
             pat = get_pat(field)
             if perm_name.startswith(pat):
                 return True
         return False
 
     def _clear_object_perm(self, object_perms, preserve=None):
         preserve = preserve or []
         _deleted = []
         for perm in object_perms:
             perm_name = perm.permission.permission_name
             if not self._keep_perm(perm_name, keep_fields=preserve):
                 _deleted.append(perm_name)
                 self.sa.delete(perm)
         return _deleted
 
     def _clear_user_perms(self, user_id, preserve=None):
         perms = self.sa.query(UserToPerm)\
             .filter(UserToPerm.user_id == user_id)\
             .all()
         return self._clear_object_perm(perms, preserve=preserve)
 
     def _clear_user_group_perms(self, user_group_id, preserve=None):
         perms = self.sa.query(UserGroupToPerm)\
             .filter(UserGroupToPerm.users_group_id == user_group_id)\
             .all()
         return self._clear_object_perm(perms, preserve=preserve)
 
-    def _set_new_object_perms(self, obj_type, object, form_result, preserve=None):
+    def _set_new_object_perms(self, obj_type, to_object, form_result, preserve=None):
         # clear current entries, to make this function idempotent
         # it will fix even if we define more permissions or permissions
         # are somehow missing
         preserve = preserve or []
         _global_perms = self.global_perms.copy()
         if obj_type not in ['user', 'user_group']:
             raise ValueError("obj_type must be on of 'user' or 'user_group'")
         global_perms = len(_global_perms)
         default_user_perms = len(Permission.DEFAULT_USER_PERMISSIONS)
         if global_perms != default_user_perms:
             raise Exception(
                 'Inconsistent permissions definition. Got {} vs {}'.format(
                     global_perms, default_user_perms))
 
         if obj_type == 'user':
-            self._clear_user_perms(object.user_id, preserve)
+            self._clear_user_perms(to_object.user_id, preserve)
         if obj_type == 'user_group':
-            self._clear_user_group_perms(object.users_group_id, preserve)
+            self._clear_user_group_perms(to_object.users_group_id, preserve)
 
         # now kill the keys that we want to preserve from the form.
         for key in preserve:
             del _global_perms[key]
 
         for k in _global_perms.copy():
             _global_perms[k] = form_result[k]
 
         # at that stage we validate all are passed inside form_result
         for _perm_key, perm_value in _global_perms.items():
             if perm_value is None:
                 raise ValueError('Missing permission for %s' % (_perm_key,))
 
             if obj_type == 'user':
                 p = self._make_new_user_perm(object, perm_value)
                 self.sa.add(p)
             if obj_type == 'user_group':
                 p = self._make_new_user_group_perm(object, perm_value)
                 self.sa.add(p)
 
     def _set_new_user_perms(self, user, form_result, preserve=None):
         return self._set_new_object_perms(
             'user', user, form_result, preserve)
 
     def _set_new_user_group_perms(self, user_group, form_result, preserve=None):
         return self._set_new_object_perms(
             'user_group', user_group, form_result, preserve)
 
     def set_new_user_perms(self, user, form_result):
         # calculate what to preserve from what is given in form_result
         preserve = set(self.global_perms.keys()).difference(set(form_result.keys()))
         return self._set_new_user_perms(user, form_result, preserve)
 
     def set_new_user_group_perms(self, user_group, form_result):
         # calculate what to preserve from what is given in form_result
         preserve = set(self.global_perms.keys()).difference(set(form_result.keys()))
         return self._set_new_user_group_perms(user_group, form_result, preserve)
 
     def create_permissions(self):
         """
         Create permissions for whole system
         """
         for p in Permission.PERMS:
             if not Permission.get_by_key(p[0]):
                 new_perm = Permission()
                 new_perm.permission_name = p[0]
                 new_perm.permission_longname = p[0]  # translation err with p[1]
                 self.sa.add(new_perm)
 
     def _create_default_object_permission(self, obj_type, obj, obj_perms,
                                           force=False):
         if obj_type not in ['user', 'user_group']:
             raise ValueError("obj_type must be on of 'user' or 'user_group'")
 
         def _get_group(perm_name):
             return '.'.join(perm_name.split('.')[:1])
 
         defined_perms_groups = map(
             _get_group, (x.permission.permission_name for x in obj_perms))
         log.debug('GOT ALREADY DEFINED:%s', obj_perms)
 
         if force:
             self._clear_object_perm(obj_perms)
             self.sa.commit()
             defined_perms_groups = []
         # for every default permission that needs to be created, we check if
         # it's group is already defined, if it's not we create default perm
         for perm_name in Permission.DEFAULT_USER_PERMISSIONS:
             gr = _get_group(perm_name)
             if gr not in defined_perms_groups:
                 log.debug('GR:%s not found, creating permission %s',
                           gr, perm_name)
                 if obj_type == 'user':
                     new_perm = self._make_new_user_perm(obj, perm_name)
                     self.sa.add(new_perm)
                 if obj_type == 'user_group':
                     new_perm = self._make_new_user_group_perm(obj, perm_name)
                     self.sa.add(new_perm)
 
     def create_default_user_permissions(self, user, force=False):
         """
         Creates only missing default permissions for user, if force is set it
         resets the default permissions for that user
 
         :param user:
         :param force:
         """
         user = self._get_user(user)
         obj_perms = UserToPerm.query().filter(UserToPerm.user == user).all()
         return self._create_default_object_permission(
             'user', user, obj_perms, force)
 
     def create_default_user_group_permissions(self, user_group, force=False):
         """
         Creates only missing default permissions for user group, if force is
         set it resets the default permissions for that user group
 
         :param user_group:
         :param force:
         """
         user_group = self._get_user_group(user_group)
         obj_perms = UserToPerm.query().filter(UserGroupToPerm.users_group == user_group).all()
         return self._create_default_object_permission(
             'user_group', user_group, obj_perms, force)
 
     def update_application_permissions(self, form_result):
         if 'perm_user_id' in form_result:
             perm_user = User.get(safe_int(form_result['perm_user_id']))
         else:
             # used mostly to do lookup for default user
             perm_user = User.get_by_username(form_result['perm_user_name'])
 
         try:
             # stage 1 set anonymous access
             if perm_user.username == User.DEFAULT_USER:
                 perm_user.active = str2bool(form_result['anonymous'])
                 self.sa.add(perm_user)
 
             # stage 2 reset defaults and set them from form data
             self._set_new_user_perms(perm_user, form_result, preserve=[
                 'default_repo_perm',
                 'default_group_perm',
                 'default_user_group_perm',
                 'default_branch_perm',
 
                 'default_repo_group_create',
                 'default_user_group_create',
                 'default_repo_create_on_write',
                 'default_repo_create',
                 'default_fork_create',
-                'default_inherit_default_permissions',])
+                'default_inherit_default_permissions'])
 
             self.sa.commit()
         except (DatabaseError,):
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update_user_permissions(self, form_result):
         if 'perm_user_id' in form_result:
             perm_user = User.get(safe_int(form_result['perm_user_id']))
         else:
             # used mostly to do lookup for default user
             perm_user = User.get_by_username(form_result['perm_user_name'])
         try:
             # stage 2 reset defaults and set them from form data
             self._set_new_user_perms(perm_user, form_result, preserve=[
                 'default_repo_perm',
                 'default_group_perm',
                 'default_user_group_perm',
                 'default_branch_perm',
 
                 'default_register',
                 'default_password_reset',
                 'default_extern_activate'])
             self.sa.commit()
         except (DatabaseError,):
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update_user_group_permissions(self, form_result):
         if 'perm_user_group_id' in form_result:
             perm_user_group = UserGroup.get(safe_int(form_result['perm_user_group_id']))
         else:
             # used mostly to do lookup for default user
             perm_user_group = UserGroup.get_by_group_name(form_result['perm_user_group_name'])
         try:
             # stage 2 reset defaults and set them from form data
             self._set_new_user_group_perms(perm_user_group, form_result, preserve=[
                 'default_repo_perm',
                 'default_group_perm',
                 'default_user_group_perm',
                 'default_branch_perm',
 
                 'default_register',
                 'default_password_reset',
                 'default_extern_activate'])
             self.sa.commit()
         except (DatabaseError,):
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update_object_permissions(self, form_result):
         if 'perm_user_id' in form_result:
             perm_user = User.get(safe_int(form_result['perm_user_id']))
         else:
             # used mostly to do lookup for default user
             perm_user = User.get_by_username(form_result['perm_user_name'])
         try:
 
             # stage 2 reset defaults and set them from form data
             self._set_new_user_perms(perm_user, form_result, preserve=[
                 'default_repo_group_create',
                 'default_user_group_create',
                 'default_repo_create_on_write',
                 'default_repo_create',
                 'default_fork_create',
                 'default_inherit_default_permissions',
                 'default_branch_perm',
 
                 'default_register',
                 'default_password_reset',
                 'default_extern_activate'])
 
             # overwrite default repo permissions
             if form_result['overwrite_default_repo']:
                 _def_name = form_result['default_repo_perm'].split('repository.')[-1]
                 _def = Permission.get_by_key('repository.' + _def_name)
                 for r2p in self.sa.query(UserRepoToPerm)\
                                .filter(UserRepoToPerm.user == perm_user)\
                                .all():
                     # don't reset PRIVATE repositories
                     if not r2p.repository.private:
                         r2p.permission = _def
                         self.sa.add(r2p)
 
             # overwrite default repo group permissions
             if form_result['overwrite_default_group']:
                 _def_name = form_result['default_group_perm'].split('group.')[-1]
                 _def = Permission.get_by_key('group.' + _def_name)
                 for g2p in self.sa.query(UserRepoGroupToPerm)\
                                .filter(UserRepoGroupToPerm.user == perm_user)\
                                .all():
                     g2p.permission = _def
                     self.sa.add(g2p)
 
             # overwrite default user group permissions
             if form_result['overwrite_default_user_group']:
                 _def_name = form_result['default_user_group_perm'].split('usergroup.')[-1]
                 # user groups
                 _def = Permission.get_by_key('usergroup.' + _def_name)
                 for g2p in self.sa.query(UserUserGroupToPerm)\
                                .filter(UserUserGroupToPerm.user == perm_user)\
                                .all():
                     g2p.permission = _def
                     self.sa.add(g2p)
 
             # COMMIT
             self.sa.commit()
         except (DatabaseError,):
             log.exception('Failed to set default object permissions')
             self.sa.rollback()
             raise
 
     def update_branch_permissions(self, form_result):
         if 'perm_user_id' in form_result:
             perm_user = User.get(safe_int(form_result['perm_user_id']))
         else:
             # used mostly to do lookup for default user
             perm_user = User.get_by_username(form_result['perm_user_name'])
         try:
 
             # stage 2 reset defaults and set them from form data
             self._set_new_user_perms(perm_user, form_result, preserve=[
                 'default_repo_perm',
                 'default_group_perm',
                 'default_user_group_perm',
 
                 'default_repo_group_create',
                 'default_user_group_create',
                 'default_repo_create_on_write',
                 'default_repo_create',
                 'default_fork_create',
                 'default_inherit_default_permissions',
 
                 'default_register',
                 'default_password_reset',
                 'default_extern_activate'])
 
             # overwrite default branch permissions
             if form_result['overwrite_default_branch']:
                 _def_name = \
                     form_result['default_branch_perm'].split('branch.')[-1]
 
                 _def = Permission.get_by_key('branch.' + _def_name)
 
                 user_perms = UserToRepoBranchPermission.query()\
                     .join(UserToRepoBranchPermission.user_repo_to_perm)\
                     .filter(UserRepoToPerm.user == perm_user).all()
 
                 for g2p in user_perms:
                     g2p.permission = _def
                     self.sa.add(g2p)
 
             # COMMIT
             self.sa.commit()
         except (DatabaseError,):
             log.exception('Failed to set default branch permissions')
             self.sa.rollback()
             raise
 
     def get_users_with_repo_write(self, db_repo):
         write_plus = ['repository.write', 'repository.admin']
         default_user_id = User.get_default_user_id()
         user_write_permissions = collections.OrderedDict()
 
         # write or higher and DEFAULT user for inheritance
         for perm in db_repo.permissions():
             if perm.permission in write_plus or perm.user_id == default_user_id:
                 user_write_permissions[perm.user_id] = perm
         return user_write_permissions
 
     def get_user_groups_with_repo_write(self, db_repo):
         write_plus = ['repository.write', 'repository.admin']
         user_group_write_permissions = collections.OrderedDict()
 
         # write or higher and DEFAULT user for inheritance
         for p in db_repo.permission_user_groups():
             if p.permission in write_plus:
                 user_group_write_permissions[p.users_group_id] = p
         return user_group_write_permissions
 
     def trigger_permission_flush(self, affected_user_ids=None):
         affected_user_ids = affected_user_ids or User.get_all_user_ids()
         events.trigger(events.UserPermissionsChange(affected_user_ids))
 
     def flush_user_permission_caches(self, changes, affected_user_ids=None):
         affected_user_ids = affected_user_ids or []
 
         for change in changes['added'] + changes['updated'] + changes['deleted']:
             if change['type'] == 'user':
                 affected_user_ids.append(change['id'])
             if change['type'] == 'user_group':
                 user_group = UserGroup.get(safe_int(change['id']))
                 if user_group:
                     group_members_ids = [x.user_id for x in user_group.members]
                     affected_user_ids.extend(group_members_ids)
 
         self.trigger_permission_flush(affected_user_ids)
 
         return affected_user_ids

rhodecode/model/scm.py

@@ -1,1028 +1,1028 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Scm model for RhodeCode
 """
 
 import os.path
 import traceback
 import logging
-from io import StringIO
+import io
 
 from sqlalchemy import func
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib import helpers as h, rc_cache
 from rhodecode.lib.auth import (
     HasRepoPermissionAny, HasRepoGroupPermissionAny,
     HasUserGroupPermissionAny)
 from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
 from rhodecode.lib import hooks_utils
 from rhodecode.lib.utils import (
     get_filesystem_repos, make_db_config)
 from rhodecode.lib.utils2 import (safe_str, safe_unicode)
 from rhodecode.lib.system_info import get_system_info
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     or_, false,
     Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
     PullRequest, FileStore)
 from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.model.validation_schema.validators import url_validator, InvalidCloneUrl
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class SimpleCachedRepoList(object):
     """
     Lighter version of of iteration of repos without the scm initialisation,
     and with cache usage
     """
     def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
         if not perm_set:
             perm_set = ['repository.read', 'repository.write',
                         'repository.admin']
         self.perm_set = perm_set
 
     def __len__(self):
         return len(self.db_repo_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             has_perm = HasRepoPermissionAny(*self.perm_set)(
                 dbr.repo_name, 'SimpleCachedRepoList check')
             if not has_perm:
                 continue
 
             tmp_d = {
                 'name': dbr.repo_name,
                 'dbrepo': dbr.get_dict(),
                 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
             }
             yield tmp_d
 
 
 class _PermCheckIterator(object):
 
     def __init__(
             self, obj_list, obj_attr, perm_set, perm_checker,
             extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
 
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker(*self.perm_set)
         self.extra_kwargs = extra_kwargs or {}
 
     def __len__(self):
         return len(self.obj_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             # NOTE(marcink): the __dict__.get() is ~4x faster then getattr()
             name = db_obj.__dict__.get(self.obj_attr, None)
             if not self.perm_checker(name, self.__class__.__name__, **self.extra_kwargs):
                 continue
 
             yield db_obj
 
 
 class RepoList(_PermCheckIterator):
 
     def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['repository.read', 'repository.write', 'repository.admin']
 
         super(RepoList, self).__init__(
             obj_list=db_repo_list,
             obj_attr='_repo_name', perm_set=perm_set,
             perm_checker=HasRepoPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class RepoGroupList(_PermCheckIterator):
 
     def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['group.read', 'group.write', 'group.admin']
 
         super(RepoGroupList, self).__init__(
             obj_list=db_repo_group_list,
             obj_attr='_group_name', perm_set=perm_set,
             perm_checker=HasRepoGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class UserGroupList(_PermCheckIterator):
 
     def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
         super(UserGroupList, self).__init__(
             obj_list=db_user_group_list,
             obj_attr='users_group_name', perm_set=perm_set,
             perm_checker=HasUserGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class ScmModel(BaseModel):
     """
     Generic Scm Model
     """
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         settings_model = VcsSettingsModel(sa=self.sa)
         return settings_model.get_repos_location()
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s', repos_path)
 
         config = make_db_config()
         config.set('extensions', 'largefiles', '')
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for rhodecode
             name = Repository.normalize_repo_name(name)
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 elif path[0] in rhodecode.BACKENDS:
                     backend = get_backend(path[0])
                     repos[name] = backend(path[1], config=config,
                                           with_wire={"cache": False})
             except OSError:
                 continue
             except RepositoryError:
                 log.exception('Failed to create a repo')
                 continue
 
         log.debug('found %s paths with repositories', len(repos))
         return repos
 
     def get_repos(self, all_repos=None, sort_key=None):
         """
         Get all repositories from db and for each repo create it's
         backend instance and fill that backed with information from database
 
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
 
         :param sort_key: initial sorting of repositories
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.group_id == None)\
                 .order_by(func.lower(Repository.repo_name)).all()
         repo_iter = SimpleCachedRepoList(
             all_repos, repos_path=self.repos_path, order_by=sort_key)
         return repo_iter
 
     def get_repo_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         return [x for x in RepoGroupList(all_groups)]
 
     def mark_for_invalidation(self, repo_name, delete=False):
         """
         Mark caches of this repo invalid in the database. `delete` flag
         removes the cache entries
 
         :param repo_name: the repo_name for which caches should be marked
             invalid, or deleted
         :param delete: delete the entry keys instead of setting bool
             flag on them, and also purge caches used by the dogpile
         """
         repo = Repository.get_by_repo_name(repo_name)
 
         if repo:
             invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
                 repo_id=repo.repo_id)
             CacheKey.set_invalidate(invalidation_namespace, delete=delete)
 
             repo_id = repo.repo_id
             config = repo._config
             config.set('extensions', 'largefiles', '')
             repo.update_commit_cache(config=config, cs_cache=None)
             if delete:
                 cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
                 rc_cache.clear_cache_namespace(
                     'cache_repo', cache_namespace_uid, invalidate=True)
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = self._get_repo(repo)
 
         return self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(Repository)\
             .filter(Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(PullRequest)\
             .filter(PullRequest.target_repo == repo)\
             .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
 
     def get_artifacts(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(FileStore)\
             .filter(FileStore.repo == repo)\
             .filter(or_(FileStore.hidden == None, FileStore.hidden == false())).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self._get_repo(repo)
         fork = self._get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
 
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError(
                 "Cannot set repository as fork of repository with other type")
 
         repo.fork = fork
         self.sa.add(repo)
         return repo
 
     def pull_changes(self, repo, username, remote_uri=None, validate_uri=True):
         dbrepo = self._get_repo(repo)
         remote_uri = remote_uri or dbrepo.clone_uri
         if not remote_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance(cache=False)
         repo.config.clear_section('hooks')
 
         try:
             # NOTE(marcink): add extra validation so we skip invalid urls
             # this is due this tasks can be executed via scheduler without
             # proper validation of remote_uri
             if validate_uri:
                 config = make_db_config(clear_session=False)
                 url_validator(remote_uri, dbrepo.repo_type, config)
         except InvalidCloneUrl:
             raise
 
         repo_name = dbrepo.repo_name
         try:
             # TODO: we need to make sure those operations call proper hooks !
             repo.fetch(remote_uri)
 
             self.mark_for_invalidation(repo_name)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def push_changes(self, repo, username, remote_uri=None, validate_uri=True):
         dbrepo = self._get_repo(repo)
         remote_uri = remote_uri or dbrepo.push_uri
         if not remote_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance(cache=False)
         repo.config.clear_section('hooks')
 
         try:
             # NOTE(marcink): add extra validation so we skip invalid urls
             # this is due this tasks can be executed via scheduler without
             # proper validation of remote_uri
             if validate_uri:
                 config = make_db_config(clear_session=False)
                 url_validator(remote_uri, dbrepo.repo_type, config)
         except InvalidCloneUrl:
             raise
 
         try:
             repo.push(remote_uri)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, commit, user, author, message,
                       content, f_path):
         """
         Commits changes
 
         :param repo: SCM instance
 
         """
         user = self._get_user(user)
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         imc = repo.in_memory_commit
         imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
         try:
             # TODO: handle pre-push action !
             tip = imc.commit(
                 message=message, author=author, parents=[commit],
                 branch=commit.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo_name)
 
         # We trigger the post-push action
         hooks_utils.trigger_post_push_hook(
             username=user.username, action='push_local', hook_type='post_push',
             repo_name=repo_name, repo_type=repo.alias, commit_ids=[tip.raw_id])
         return tip
 
     def _sanitize_path(self, f_path):
         if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = os.path.normpath(f_path)
         return f_path
 
     def get_dirnode_metadata(self, request, commit, dir_node):
         if not dir_node.is_dir():
             return []
 
         data = []
         for node in dir_node:
             if not node.is_file():
                 # we skip file-nodes
                 continue
 
             last_commit = node.last_commit
             last_commit_date = last_commit.date
             data.append({
                 'name': node.name,
                 'size': h.format_byte_size_binary(node.size),
                 'modified_at': h.format_date(last_commit_date),
                 'modified_ts': last_commit_date.isoformat(),
                 'revision': last_commit.revision,
                 'short_id': last_commit.short_id,
                 'message': h.escape(last_commit.message),
                 'author': h.escape(last_commit.author),
                 'user_profile': h.gravatar_with_user(
                     request, last_commit.author),
             })
 
         return data
 
     def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
                   extended_info=False, content=False, max_file_bytes=None):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
 
         :param repo_name: name of repository
         :param commit_id: commit id for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
         :param extended_info: show additional info such as md5, binary, size etc
         :param content: add nodes content to the return data
         :param max_file_bytes: will not return file contents over this limit
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
             for __, dirs, files in commit.walk(root_path):
 
                 for f in files:
                     _content = None
                     _data = f_name = f.unicode_path
 
                     if not flat:
                         _data = {
                             "name": h.escape(f_name),
                             "type": "file",
                             }
                         if extended_info:
                             _data.update({
                                 "md5": f.md5,
                                 "binary": f.is_binary,
                                 "size": f.size,
                                 "extension": f.extension,
                                 "mimetype": f.mimetype,
                                 "lines": f.lines()[0]
                             })
 
                         if content:
                             over_size_limit = (max_file_bytes is not None
                                                and f.size > max_file_bytes)
                             full_content = None
                             if not f.is_binary and not over_size_limit:
                                 full_content = safe_str(f.content)
 
                             _data.update({
                                 "content": full_content,
                             })
                     _files.append(_data)
 
                 for d in dirs:
                     _data = d_name = d.unicode_path
                     if not flat:
                         _data = {
                             "name": h.escape(d_name),
                             "type": "dir",
                             }
                     if extended_info:
                         _data.update({
                             "md5": None,
                             "binary": None,
                             "size": None,
                             "extension": None,
                         })
                     if content:
                         _data.update({
                             "content": None
                         })
                     _dirs.append(_data)
         except RepositoryError:
             log.exception("Exception in get_nodes")
             raise
 
         return _dirs, _files
 
     def get_quick_filter_nodes(self, repo_name, commit_id, root_path='/'):
         """
         Generate files for quick filter in files view
         """
 
         _files = list()
         _dirs = list()
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
             for __, dirs, files in commit.walk(root_path):
 
                 for f in files:
 
                     _data = {
                         "name": h.escape(f.unicode_path),
                         "type": "file",
                         }
 
                     _files.append(_data)
 
                 for d in dirs:
 
                     _data = {
                         "name": h.escape(d.unicode_path),
                         "type": "dir",
                         }
 
                     _dirs.append(_data)
         except RepositoryError:
             log.exception("Exception in get_quick_filter_nodes")
             raise
 
         return _dirs, _files
 
     def get_node(self, repo_name, commit_id, file_path,
                  extended_info=False, content=False, max_file_bytes=None, cache=True):
         """
         retrieve single node from commit
         """
         try:
 
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
 
             file_node = commit.get_node(file_path)
             if file_node.is_dir():
                 raise RepositoryError('The given path is a directory')
 
             _content = None
             f_name = file_node.unicode_path
 
             file_data = {
                 "name": h.escape(f_name),
                 "type": "file",
             }
 
             if extended_info:
                 file_data.update({
                     "extension": file_node.extension,
                     "mimetype": file_node.mimetype,
                 })
 
                 if cache:
                     md5 = file_node.md5
                     is_binary = file_node.is_binary
                     size = file_node.size
                 else:
                     is_binary, md5, size, _content = file_node.metadata_uncached()
 
                 file_data.update({
                     "md5": md5,
                     "binary": is_binary,
                     "size": size,
                 })
 
             if content and cache:
                 # get content + cache
                 size = file_node.size
                 over_size_limit = (max_file_bytes is not None and size > max_file_bytes)
                 full_content = None
                 all_lines = 0
                 if not file_node.is_binary and not over_size_limit:
                     full_content = safe_unicode(file_node.content)
                     all_lines, empty_lines = file_node.count_lines(full_content)
 
                 file_data.update({
                     "content": full_content,
                     "lines": all_lines
                 })
             elif content:
                 # get content *without* cache
                 if _content is None:
                     is_binary, md5, size, _content = file_node.metadata_uncached()
 
                 over_size_limit = (max_file_bytes is not None and size > max_file_bytes)
                 full_content = None
                 all_lines = 0
                 if not is_binary and not over_size_limit:
                     full_content = safe_unicode(_content)
                     all_lines, empty_lines = file_node.count_lines(full_content)
 
                 file_data.update({
                     "content": full_content,
                     "lines": all_lines
                 })
 
         except RepositoryError:
             log.exception("Exception in get_node")
             raise
 
         return file_data
 
     def get_fts_data(self, repo_name, commit_id, root_path='/'):
         """
         Fetch node tree for usage in full text search
         """
 
         tree_info = list()
 
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
             for __, dirs, files in commit.walk(root_path):
 
                 for f in files:
                     is_binary, md5, size, _content = f.metadata_uncached()
                     _data = {
                         "name": f.unicode_path,
                         "md5": md5,
                         "extension": f.extension,
                         "binary": is_binary,
                         "size": size
                     }
 
                     tree_info.append(_data)
 
         except RepositoryError:
             log.exception("Exception in get_nodes")
             raise
 
         return tree_info
 
     def create_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Commits given multiple nodes into repo
 
         :param user: RhodeCode User object or user_id, the commiter
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's
            initial commit
         :param author: author of commit, cna be different that commiter
            only for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new committed commit
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             content = nodes[f_path]['content']
             f_path = safe_str(f_path)
             # decoding here will force that we have proper encoded values
             # in any other case this will throw exceptions and deny commit
             if isinstance(content, (str,)):
                 content = safe_str(content)
             elif isinstance(content, (file, cStringIO.OutputType,)):
                 content = content.read()
             else:
                 raise Exception('Content is of unrecognized type %s' % (
                     type(content)
                 ))
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_type=scm_instance.alias,
                 hook_type='post_push',
                 commit_ids=[tip.raw_id])
         return tip
 
     def update_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
 
         # add multiple nodes
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one, also sanitaze
             # the path for any hack around relative paths like ../../ etc.
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
             file_mode = data.get('mode')
             filenode = FileNode(old_filename, content=content, mode=file_mode)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames more efficient, needs vcs lib changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content, mode=file_mode))
                 else:
                     imc.change(filenode)
 
         try:
             # TODO: handle pre push scenario commit changes
             tip = imc.commit(message=message,
                              author=author,
                              parents=parents,
                              branch=parent_commit.branch)
         except NodeNotChangedError:
             raise
         except Exception as e:
             log.exception("Unexpected exception during call to imc.commit")
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo.repo_name)
 
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local', hook_type='post_push',
                 repo_name=repo.repo_name, repo_type=scm_instance.alias,
                 commit_ids=[tip.raw_id])
 
         return tip
 
     def delete_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes given multiple nodes into `repo`
 
         :param user: RhodeCode User object or user_id, the committer
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's initial
            commit
         :param author: author of commit, cna be different that commiter only
            for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new commit after deletion
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatabilty it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
 
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local', hook_type='post_push',
                 repo_name=repo.repo_name, repo_type=scm_instance.alias,
                 commit_ids=[tip.raw_id])
         return tip
 
     def strip(self, repo, commit_id, branch):
         scm_instance = repo.scm_instance(cache=False)
         scm_instance.config.clear_section('hooks')
         scm_instance.strip(commit_id, branch)
         self.mark_for_invalidation(repo.repo_name)
 
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
 
     @classmethod
     def backend_landing_ref(cls, repo_type):
         """
         Return a default landing ref based on a repository type.
         """
 
         landing_ref = {
             'hg': ('branch:default', 'default'),
             'git': ('branch:master', 'master'),
             'svn': ('rev:tip', 'latest tip'),
             'default': ('rev:tip', 'latest tip'),
         }
 
         return landing_ref.get(repo_type) or landing_ref['default']
 
     def get_repo_landing_revs(self, translator, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
         """
         from rhodecode.lib.vcs.backends.git import GitRepository
 
         _ = translator
         repo = self._get_repo(repo)
 
         if repo:
             repo_type = repo.repo_type
         else:
             repo_type = 'default'
 
         default_landing_ref, landing_ref_lbl = self.backend_landing_ref(repo_type)
 
         default_ref_options = [
             [default_landing_ref, landing_ref_lbl]
         ]
         default_choices = [
             default_landing_ref
         ]
 
         if not repo:
             # presented at NEW repo creation
             return default_choices, default_ref_options
 
         repo = repo.scm_instance()
 
         ref_options = [(default_landing_ref, landing_ref_lbl)]
         choices = [default_landing_ref]
 
         # branches
         branch_group = [(u'branch:%s' % safe_unicode(b), safe_unicode(b)) for b in repo.branches]
         if not branch_group:
             # new repo, or without maybe a branch?
             branch_group = default_ref_options
 
         branches_group = (branch_group, _("Branches"))
         ref_options.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         # bookmarks for HG
         if repo.alias == 'hg':
             bookmarks_group = (
                 [(u'book:%s' % safe_unicode(b), safe_unicode(b))
                     for b in repo.bookmarks],
                 _("Bookmarks"))
             ref_options.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         # tags
         tags_group = (
             [(u'tag:%s' % safe_unicode(t), safe_unicode(t))
                 for t in repo.tags],
             _("Tags"))
         ref_options.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, ref_options
 
     def get_server_info(self, environ=None):
         server_info = get_system_info(environ)
         return server_info

rhodecode/model/validators.py

@@ -1,1115 +1,1115 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Set of generic validators
 """
 
 
 import os
 import re
 import logging
 import collections
 
 import formencode
 import ipaddress
 from formencode.validators import (
     UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set,
     NotEmpty, IPAddress, CIDR, String, FancyValidator
 )
 
 from sqlalchemy.sql.expression import true
 from sqlalchemy.util import OrderedSet
 
 from rhodecode.authentication import (
     legacy_plugin_prefix, _import_legacy_plugin)
 from rhodecode.authentication.base import loadplugin
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.auth import HasRepoGroupPermissionAny, HasPermissionAny
 from rhodecode.lib.utils import repo_name_slug, make_db_config
 from rhodecode.lib.utils2 import safe_int, str2bool, aslist, md5, safe_unicode
 from rhodecode.lib.vcs.backends.git.repository import GitRepository
 from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
 from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository
 from rhodecode.model.db import (
     RepoGroup, Repository, UserGroup, User, ChangesetStatus, Gist)
 from rhodecode.model.settings import VcsSettingsModel
 
 # silence warnings and pylint
 UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set, \
     NotEmpty, IPAddress, CIDR, String, FancyValidator
 
 log = logging.getLogger(__name__)
 
 
 class _Missing(object):
     pass
 
 
 Missing = _Missing()
 
 
 def M(self, key, state, **kwargs):
     """
     returns string from self.message based on given key,
     passed kw params are used to substitute %(named)s params inside
     translated strings
 
     :param msg:
     :param state:
     """
 
     #state._ = staticmethod(_)
     # inject validator into state object
     return self.message(key, state, **kwargs)
 
 
 def UniqueList(localizer, convert=None):
     _ = localizer
 
     class _validator(formencode.FancyValidator):
         """
         Unique List !
         """
         messages = {
-            'empty': _(u'Value cannot be an empty list'),
-            'missing_value': _(u'Value cannot be an empty list'),
+            'empty': _('Value cannot be an empty list'),
+            'missing_value': _('Value cannot be an empty list'),
         }
 
         def _to_python(self, value, state):
             ret_val = []
 
             def make_unique(value):
                 seen = []
                 return [c for c in value if not (c in seen or seen.append(c))]
 
             if isinstance(value, list):
                 ret_val = make_unique(value)
             elif isinstance(value, set):
                 ret_val = make_unique(list(value))
             elif isinstance(value, tuple):
                 ret_val = make_unique(list(value))
             elif value is None:
                 ret_val = []
             else:
                 ret_val = [value]
 
             if convert:
                 ret_val = map(convert, ret_val)
             return ret_val
 
         def empty_value(self, value):
             return []
     return _validator
 
 
 def UniqueListFromString(localizer):
     _ = localizer
 
     class _validator(UniqueList(localizer)):
         def _to_python(self, value, state):
             if isinstance(value, str):
                 value = aslist(value, ',')
             return super(_validator, self)._to_python(value, state)
     return _validator
 
 
 def ValidSvnPattern(localizer, section, repo_name=None):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'pattern_exists': _(u'Pattern already exists'),
+            'pattern_exists': _('Pattern already exists'),
         }
 
         def validate_python(self, value, state):
             if not value:
                 return
             model = VcsSettingsModel(repo=repo_name)
             ui_settings = model.get_svn_patterns(section=section)
             for entry in ui_settings:
                 if value == entry.value:
                     msg = M(self, 'pattern_exists', state)
                     raise formencode.Invalid(msg, value, state)
     return _validator
 
 
 def ValidUsername(localizer, edit=False, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'username_exists': _(u'Username "%(username)s" already exists'),
+            'username_exists': _('Username "%(username)s" already exists'),
             'system_invalid_username':
-                _(u'Username "%(username)s" is forbidden'),
+                _('Username "%(username)s" is forbidden'),
             'invalid_username':
-                _(u'Username may only contain alphanumeric characters '
-                    u'underscores, periods or dashes and must begin with '
-                    u'alphanumeric character or underscore')
+                _('Username may only contain alphanumeric characters '
+                    'underscores, periods or dashes and must begin with '
+                    'alphanumeric character or underscore')
         }
 
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 msg = M(self, 'system_invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state)
             # check if user is unique
             old_un = None
             if edit:
                 old_un = User.get(old_data.get('user_id')).username
 
             if old_un != value or not edit:
                 if User.get_by_username(value, case_insensitive=True):
                     msg = M(self, 'username_exists', state, username=value)
                     raise formencode.Invalid(msg, value, state)
 
             if (re.match(r'^[\w]{1}[\w\-\.]{0,254}$', value)
                     is None):
                 msg = M(self, 'invalid_username', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 
 
 def ValidRepoUser(localizer, allow_disabled=False):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'invalid_username': _(u'Username %(username)s is not valid'),
-            'disabled_username': _(u'Username %(username)s is disabled')
+            'invalid_username': _('Username %(username)s is not valid'),
+            'disabled_username': _('Username %(username)s is disabled')
         }
 
         def validate_python(self, value, state):
             try:
                 user = User.query().filter(User.username == value).one()
             except Exception:
                 msg = M(self, 'invalid_username', state, username=value)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'username': msg}
                 )
             if user and (not allow_disabled and not user.active):
                 msg = M(self, 'disabled_username', state, username=value)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'username': msg}
                 )
     return _validator
 
 
 def ValidUserGroup(localizer, edit=False, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'invalid_group': _(u'Invalid user group name'),
-            'group_exist': _(u'User group `%(usergroup)s` already exists'),
+            'invalid_group': _('Invalid user group name'),
+            'group_exist': _('User group `%(usergroup)s` already exists'),
             'invalid_usergroup_name':
-                _(u'user group name may only contain alphanumeric '
-                  u'characters underscores, periods or dashes and must begin '
-                  u'with alphanumeric character')
+                _('user group name may only contain alphanumeric '
+                  'characters underscores, periods or dashes and must begin '
+                  'with alphanumeric character')
         }
 
         def validate_python(self, value, state):
             if value in ['default']:
                 msg = M(self, 'invalid_group', state)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'users_group_name': msg}
                 )
             # check if group is unique
             old_ugname = None
             if edit:
                 old_id = old_data.get('users_group_id')
                 old_ugname = UserGroup.get(old_id).users_group_name
 
             if old_ugname != value or not edit:
                 is_existing_group = UserGroup.get_by_group_name(
                     value, case_insensitive=True)
                 if is_existing_group:
                     msg = M(self, 'group_exist', state, usergroup=value)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'users_group_name': msg}
                     )
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 msg = M(self, 'invalid_usergroup_name', state)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'users_group_name': msg}
                 )
     return _validator
 
 
 def ValidRepoGroup(localizer, edit=False, old_data=None, can_create_in_root=False):
     _ = localizer
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'group_parent_id': _(u'Cannot assign this group as parent'),
-            'group_exists': _(u'Group "%(group_name)s" already exists'),
-            'repo_exists': _(u'Repository with name "%(group_name)s" '
-                             u'already exists'),
-            'permission_denied': _(u"no permission to store repository group"
-                                   u"in this location"),
+            'group_parent_id': _('Cannot assign this group as parent'),
+            'group_exists': _('Group "%(group_name)s" already exists'),
+            'repo_exists': _('Repository with name "%(group_name)s" '
+                             'already exists'),
+            'permission_denied': _("no permission to store repository group"
+                                   "in this location"),
             'permission_denied_root': _(
-                u"no permission to store repository group "
-                u"in root location")
+                "no permission to store repository group "
+                "in root location")
         }
 
         def _to_python(self, value, state):
             group_name = repo_name_slug(value.get('group_name', ''))
             group_parent_id = safe_int(value.get('group_parent_id'))
             gr = RepoGroup.get(group_parent_id)
             if gr:
                 parent_group_path = gr.full_path
                 # value needs to be aware of group name in order to check
                 # db key This is an actual just the name to store in the
                 # database
                 group_name_full = (
                     parent_group_path + RepoGroup.url_sep() + group_name)
             else:
                 group_name_full = group_name
 
             value['group_name'] = group_name
             value['group_name_full'] = group_name_full
             value['group_parent_id'] = group_parent_id
             return value
 
         def validate_python(self, value, state):
 
             old_group_name = None
             group_name = value.get('group_name')
             group_name_full = value.get('group_name_full')
             group_parent_id = safe_int(value.get('group_parent_id'))
             if group_parent_id == -1:
                 group_parent_id = None
 
             group_obj = RepoGroup.get(old_data.get('group_id'))
             parent_group_changed = False
             if edit:
                 old_group_name = group_obj.group_name
                 old_group_parent_id = group_obj.group_parent_id
 
                 if group_parent_id != old_group_parent_id:
                     parent_group_changed = True
 
                 # TODO: mikhail: the following if statement is not reached
                 # since group_parent_id's OneOf validation fails before.
                 # Can be removed.
 
                 # check against setting a parent of self
                 parent_of_self = (
                     old_data['group_id'] == group_parent_id
                     if group_parent_id else False
                 )
                 if parent_of_self:
                     msg = M(self, 'group_parent_id', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'group_parent_id': msg}
                     )
 
             # group we're moving current group inside
             child_group = None
             if group_parent_id:
                 child_group = RepoGroup.query().filter(
                     RepoGroup.group_id == group_parent_id).scalar()
 
             # do a special check that we cannot move a group to one of
             # it's children
             if edit and child_group:
                 parents = [x.group_id for x in child_group.parents]
                 move_to_children = old_data['group_id'] in parents
                 if move_to_children:
                     msg = M(self, 'group_parent_id', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'group_parent_id': msg})
 
             # Check if we have permission to store in the parent.
             # Only check if the parent group changed.
             if parent_group_changed:
                 if child_group is None:
                     if not can_create_in_root:
                         msg = M(self, 'permission_denied_root', state)
                         raise formencode.Invalid(
                             msg, value, state,
                             error_dict={'group_parent_id': msg})
                 else:
                     valid = HasRepoGroupPermissionAny('group.admin')
                     forbidden = not valid(
                         child_group.group_name, 'can create group validator')
                     if forbidden:
                         msg = M(self, 'permission_denied', state)
                         raise formencode.Invalid(
                             msg, value, state,
                             error_dict={'group_parent_id': msg})
 
             # if we change the name or it's new group, check for existing names
             # or repositories with the same name
             if old_group_name != group_name_full or not edit:
                 # check group
                 gr = RepoGroup.get_by_group_name(group_name_full)
                 if gr:
                     msg = M(self, 'group_exists', state, group_name=group_name)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'group_name': msg})
 
                 # check for same repo
                 repo = Repository.get_by_repo_name(group_name_full)
                 if repo:
                     msg = M(self, 'repo_exists', state, group_name=group_name)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'group_name': msg})
     return _validator
 
 
 def ValidPassword(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password':
-                _(u'Invalid characters (non-ascii) in password')
+                _('Invalid characters (non-ascii) in password')
         }
 
         def validate_python(self, value, state):
             try:
                 (value or '').decode('ascii')
             except UnicodeError:
                 msg = M(self, 'invalid_password', state)
                 raise formencode.Invalid(msg, value, state,)
     return _validator
 
 
 def ValidPasswordsMatch(
         localizer, passwd='new_password',
         passwd_confirmation='password_confirmation'):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'password_mismatch': _(u'Passwords do not match'),
+            'password_mismatch': _('Passwords do not match'),
         }
 
         def validate_python(self, value, state):
 
             pass_val = value.get('password') or value.get(passwd)
             if pass_val != value[passwd_confirmation]:
                 msg = M(self, 'password_mismatch', state)
                 raise formencode.Invalid(
                     msg, value, state,
                     error_dict={passwd: msg, passwd_confirmation: msg}
                 )
     return _validator
 
 
 def ValidAuth(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'invalid_password': _(u'invalid password'),
-            'invalid_username': _(u'invalid user name'),
-            'disabled_account': _(u'Your account is disabled')
+            'invalid_password': _('invalid password'),
+            'invalid_username': _('invalid user name'),
+            'disabled_account': _('Your account is disabled')
         }
 
         def validate_python(self, value, state):
             from rhodecode.authentication.base import authenticate, HTTP_TYPE
 
             password = value['password']
             username = value['username']
 
             if not authenticate(username, password, '', HTTP_TYPE,
                                 skip_missing=True):
                 user = User.get_by_username(username)
                 if user and not user.active:
                     log.warning('user %s is disabled', username)
                     msg = M(self, 'disabled_account', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'username': msg}
                     )
                 else:
                     log.warning('user `%s` failed to authenticate', username)
                     msg = M(self, 'invalid_username', state)
                     msg2 = M(self, 'invalid_password', state)
                     raise formencode.Invalid(
                         msg, value, state,
                         error_dict={'username': msg, 'password': msg2}
                     )
     return _validator
 
 
 def ValidRepoName(localizer, edit=False, old_data=None):
     old_data = old_data or {}
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_repo_name':
-                _(u'Repository name %(repo)s is disallowed'),
+                _('Repository name %(repo)s is disallowed'),
             # top level
-            'repository_exists': _(u'Repository with name %(repo)s '
-                                   u'already exists'),
-            'group_exists': _(u'Repository group with name "%(repo)s" '
-                              u'already exists'),
+            'repository_exists': _('Repository with name %(repo)s '
+                                   'already exists'),
+            'group_exists': _('Repository group with name "%(repo)s" '
+                              'already exists'),
             # inside a group
-            'repository_in_group_exists': _(u'Repository with name %(repo)s '
-                                            u'exists in group "%(group)s"'),
+            'repository_in_group_exists': _('Repository with name %(repo)s '
+                                            'exists in group "%(group)s"'),
             'group_in_group_exists': _(
-                u'Repository group with name "%(repo)s" '
-                u'exists in group "%(group)s"'),
+                'Repository group with name "%(repo)s" '
+                'exists in group "%(group)s"'),
         }
 
         def _to_python(self, value, state):
             repo_name = repo_name_slug(value.get('repo_name', ''))
             repo_group = value.get('repo_group')
             if repo_group:
                 gr = RepoGroup.get(repo_group)
                 group_path = gr.full_path
                 group_name = gr.group_name
                 # value needs to be aware of group name in order to check
                 # db key This is an actual just the name to store in the
                 # database
                 repo_name_full = group_path + RepoGroup.url_sep() + repo_name
             else:
                 group_name = group_path = ''
                 repo_name_full = repo_name
 
             value['repo_name'] = repo_name
             value['repo_name_full'] = repo_name_full
             value['group_path'] = group_path
             value['group_name'] = group_name
             return value
 
         def validate_python(self, value, state):
 
             repo_name = value.get('repo_name')
             repo_name_full = value.get('repo_name_full')
             group_path = value.get('group_path')
             group_name = value.get('group_name')
 
             if repo_name in [ADMIN_PREFIX, '']:
                 msg = M(self, 'invalid_repo_name', state, repo=repo_name)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'repo_name': msg})
 
             rename = old_data.get('repo_name') != repo_name_full
             create = not edit
             if rename or create:
 
                 if group_path:
                     if Repository.get_by_repo_name(repo_name_full):
                         msg = M(self, 'repository_in_group_exists', state,
                                 repo=repo_name, group=group_name)
                         raise formencode.Invalid(
                             msg, value, state, error_dict={'repo_name': msg})
                     if RepoGroup.get_by_group_name(repo_name_full):
                         msg = M(self, 'group_in_group_exists', state,
                                 repo=repo_name, group=group_name)
                         raise formencode.Invalid(
                             msg, value, state, error_dict={'repo_name': msg})
                 else:
                     if RepoGroup.get_by_group_name(repo_name_full):
                         msg = M(self, 'group_exists', state, repo=repo_name)
                         raise formencode.Invalid(
                             msg, value, state, error_dict={'repo_name': msg})
 
                     if Repository.get_by_repo_name(repo_name_full):
                         msg = M(
                             self, 'repository_exists', state, repo=repo_name)
                         raise formencode.Invalid(
                             msg, value, state, error_dict={'repo_name': msg})
             return value
     return _validator
 
 
 def ValidForkName(localizer, *args, **kwargs):
     _ = localizer
 
     return ValidRepoName(localizer, *args, **kwargs)
 
 
 def SlugifyName(localizer):
     _ = localizer
     
     class _validator(formencode.validators.FancyValidator):
 
         def _to_python(self, value, state):
             return repo_name_slug(value)
 
         def validate_python(self, value, state):
             pass
     return _validator
 
 
 def CannotHaveGitSuffix(localizer):
     _ = localizer
     
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'has_git_suffix':
-                _(u'Repository name cannot end with .git'),
+                _('Repository name cannot end with .git'),
         }
 
         def _to_python(self, value, state):
             return value
 
         def validate_python(self, value, state):
             if value and value.endswith('.git'):
                 msg = M(
                     self, 'has_git_suffix', state)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'repo_name': msg})
     return _validator
 
 
 def ValidCloneUri(localizer):
     _ = localizer
     
     class InvalidCloneUrl(Exception):
         allowed_prefixes = ()
 
     def url_handler(repo_type, url):
         config = make_db_config(clear_session=False)
         if repo_type == 'hg':
             allowed_prefixes = ('http', 'svn+http', 'git+http')
 
             if 'http' in url[:4]:
                 # initially check if it's at least the proper URL
                 # or does it pass basic auth
                 MercurialRepository.check_url(url, config)
             elif 'svn+http' in url[:8]:  # svn->hg import
                 SubversionRepository.check_url(url, config)
             elif 'git+http' in url[:8]:  # git->hg import
                 raise NotImplementedError()
             else:
                 exc = InvalidCloneUrl('Clone from URI %s not allowed. '
                                       'Allowed url must start with one of %s'
                                       % (url, ','.join(allowed_prefixes)))
                 exc.allowed_prefixes = allowed_prefixes
                 raise exc
 
         elif repo_type == 'git':
             allowed_prefixes = ('http', 'svn+http', 'hg+http')
             if 'http' in url[:4]:
                 # initially check if it's at least the proper URL
                 # or does it pass basic auth
                 GitRepository.check_url(url, config)
             elif 'svn+http' in url[:8]:  # svn->git import
                 raise NotImplementedError()
             elif 'hg+http' in url[:8]:  # hg->git import
                 raise NotImplementedError()
             else:
                 exc = InvalidCloneUrl('Clone from URI %s not allowed. '
                                       'Allowed url must start with one of %s'
                                       % (url, ','.join(allowed_prefixes)))
                 exc.allowed_prefixes = allowed_prefixes
                 raise exc
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'clone_uri': _(u'invalid clone url or credentials for %(rtype)s repository'),
+            'clone_uri': _('invalid clone url or credentials for %(rtype)s repository'),
             'invalid_clone_uri': _(
-                u'Invalid clone url, provide a valid clone '
-                u'url starting with one of %(allowed_prefixes)s')
+                'Invalid clone url, provide a valid clone '
+                'url starting with one of %(allowed_prefixes)s')
         }
 
         def validate_python(self, value, state):
             repo_type = value.get('repo_type')
             url = value.get('clone_uri')
 
             if url:
                 try:
                     url_handler(repo_type, url)
                 except InvalidCloneUrl as e:
                     log.warning(e)
                     msg = M(self, 'invalid_clone_uri', state, rtype=repo_type,
                             allowed_prefixes=','.join(e.allowed_prefixes))
                     raise formencode.Invalid(msg, value, state,
                                              error_dict={'clone_uri': msg})
                 except Exception:
                     log.exception('Url validation failed')
                     msg = M(self, 'clone_uri', state, rtype=repo_type)
                     raise formencode.Invalid(msg, value, state,
                                              error_dict={'clone_uri': msg})
     return _validator
 
 
 def ValidForkType(localizer, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'invalid_fork_type': _(u'Fork have to be the same type as parent')
+            'invalid_fork_type': _('Fork have to be the same type as parent')
         }
 
         def validate_python(self, value, state):
             if old_data['repo_type'] != value:
                 msg = M(self, 'invalid_fork_type', state)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'repo_type': msg}
                 )
     return _validator
 
 
 def CanWriteGroup(localizer, old_data=None):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _(
-                u"You do not have the permission "
-                u"to create repositories in this group."),
+                "You do not have the permission "
+                "to create repositories in this group."),
             'permission_denied_root': _(
-                u"You do not have the permission to store repositories in "
-                u"the root location.")
+                "You do not have the permission to store repositories in "
+                "the root location.")
         }
 
         def _to_python(self, value, state):
             # root location
             if value in [-1, "-1"]:
                 return None
             return value
 
         def validate_python(self, value, state):
             gr = RepoGroup.get(value)
             gr_name = gr.group_name if gr else None  # None means ROOT location
             # create repositories with write permission on group is set to true
             create_on_write = HasPermissionAny(
                 'hg.create.write_on_repogroup.true')()
             group_admin = HasRepoGroupPermissionAny('group.admin')(
                 gr_name, 'can write into group validator')
             group_write = HasRepoGroupPermissionAny('group.write')(
                 gr_name, 'can write into group validator')
             forbidden = not (group_admin or (group_write and create_on_write))
             can_create_repos = HasPermissionAny(
                 'hg.admin', 'hg.create.repository')
             gid = (old_data['repo_group'].get('group_id')
                    if (old_data and 'repo_group' in old_data) else None)
             value_changed = gid != safe_int(value)
             new = not old_data
             # do check if we changed the value, there's a case that someone got
             # revoked write permissions to a repository, he still created, we
             # don't need to check permission if he didn't change the value of
             # groups in form box
             if value_changed or new:
                 # parent group need to be existing
                 if gr and forbidden:
                     msg = M(self, 'permission_denied', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'repo_type': msg}
                     )
                 # check if we can write to root location !
                 elif gr is None and not can_create_repos():
                     msg = M(self, 'permission_denied_root', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'repo_type': msg}
                     )
     return _validator
 
 
 def ValidPerms(localizer, type_='repo'):
     _ = localizer
     if type_ == 'repo_group':
         EMPTY_PERM = 'group.none'
     elif type_ == 'repo':
         EMPTY_PERM = 'repository.none'
     elif type_ == 'user_group':
         EMPTY_PERM = 'usergroup.none'
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'perm_new_member_name':
-                _(u'This username or user group name is not valid')
+                _('This username or user group name is not valid')
         }
 
         def _to_python(self, value, state):
             perm_updates = OrderedSet()
             perm_additions = OrderedSet()
             perm_deletions = OrderedSet()
             # build a list of permission to update/delete and new permission
 
             # Read the perm_new_member/perm_del_member attributes and group
             # them by they IDs
             new_perms_group = collections.defaultdict(dict)
             del_perms_group = collections.defaultdict(dict)
             for k, v in value.copy().items():
                 if k.startswith('perm_del_member'):
                     # delete from org storage so we don't process that later
                     del value[k]
                     # part is `id`, `type`
                     _type, part = k.split('perm_del_member_')
                     args = part.split('_')
                     if len(args) == 2:
                         _key, pos = args
                         del_perms_group[pos][_key] = v
                 if k.startswith('perm_new_member'):
                     # delete from org storage so we don't process that later
                     del value[k]
                     # part is `id`, `type`, `perm`
                     _type, part = k.split('perm_new_member_')
                     args = part.split('_')
                     if len(args) == 2:
                         _key, pos = args
                         new_perms_group[pos][_key] = v
 
             # store the deletes
             for k in sorted(del_perms_group.keys()):
                 perm_dict = del_perms_group[k]
                 del_member = perm_dict.get('id')
                 del_type = perm_dict.get('type')
                 if del_member and del_type:
                     perm_deletions.add(
                         (del_member, None, del_type))
 
             # store additions in order of how they were added in web form
             for k in sorted(new_perms_group.keys()):
                 perm_dict = new_perms_group[k]
                 new_member = perm_dict.get('id')
                 new_type = perm_dict.get('type')
                 new_perm = perm_dict.get('perm')
                 if new_member and new_perm and new_type:
                     perm_additions.add(
                         (new_member, new_perm, new_type))
 
             # get updates of permissions
             # (read the existing radio button states)
             default_user_id = User.get_default_user_id()
 
             for k, update_value in value.items():
                 if k.startswith('u_perm_') or k.startswith('g_perm_'):
                     obj_type = k[0]
                     obj_id = k[7:]
                     update_type = {'u': 'user',
                                    'g': 'user_group'}[obj_type]
 
                     if obj_type == 'u' and safe_int(obj_id) == default_user_id:
                         if str2bool(value.get('repo_private')):
                             # prevent from updating default user permissions
                             # when this repository is marked as private
                             update_value = EMPTY_PERM
 
                     perm_updates.add(
                         (obj_id, update_value, update_type))
 
             value['perm_additions'] = []  # propagated later
             value['perm_updates'] = list(perm_updates)
             value['perm_deletions'] = list(perm_deletions)
 
             updates_map = dict(
                 (x[0], (x[1], x[2])) for x in value['perm_updates'])
             # make sure Additions don't override updates.
             for member_id, perm, member_type in list(perm_additions):
                 if member_id in updates_map:
                     perm = updates_map[member_id][0]
                 value['perm_additions'].append((member_id, perm, member_type))
 
                 # on new entries validate users they exist and they are active !
                 # this leaves feedback to the form
                 try:
                     if member_type == 'user':
                         User.query()\
                             .filter(User.active == true())\
                             .filter(User.user_id == member_id).one()
                     if member_type == 'user_group':
                         UserGroup.query()\
                             .filter(UserGroup.users_group_active == true())\
                             .filter(UserGroup.users_group_id == member_id)\
                             .one()
 
                 except Exception:
                     log.exception('Updated permission failed: org_exc:')
                     msg = M(self, 'perm_new_member_type', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={
                             'perm_new_member_name': msg}
                     )
             return value
     return _validator
 
 
 def ValidPath(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'invalid_path': _(u'This is not a valid path')
+            'invalid_path': _('This is not a valid path')
         }
 
         def validate_python(self, value, state):
             if not os.path.isdir(value):
                 msg = M(self, 'invalid_path', state)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'paths_root_path': msg}
                 )
     return _validator
 
 
 def UniqSystemEmail(localizer, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'email_taken': _(u'This e-mail address is already taken')
+            'email_taken': _('This e-mail address is already taken')
         }
 
         def _to_python(self, value, state):
             return value.lower()
 
         def validate_python(self, value, state):
             if (old_data.get('email') or '').lower() != value:
                 user = User.get_by_email(value, case_insensitive=True)
                 if user:
                     msg = M(self, 'email_taken', state)
                     raise formencode.Invalid(
                         msg, value, state, error_dict={'email': msg}
                     )
     return _validator
 
 
 def ValidSystemEmail(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'non_existing_email': _(u'e-mail "%(email)s" does not exist.')
+            'non_existing_email': _('e-mail "%(email)s" does not exist.')
         }
 
         def _to_python(self, value, state):
             return value.lower()
 
         def validate_python(self, value, state):
             user = User.get_by_email(value, case_insensitive=True)
             if user is None:
                 msg = M(self, 'non_existing_email', state, email=value)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'email': msg}
                 )
     return _validator
 
 
 def NotReviewedRevisions(localizer, repo_id):
     _ = localizer
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'rev_already_reviewed':
-                _(u'Revisions %(revs)s are already part of pull request '
-                  u'or have set status'),
+                _('Revisions %(revs)s are already part of pull request '
+                  'or have set status'),
         }
 
         def validate_python(self, value, state):
             # check revisions if they are not reviewed, or a part of another
             # pull request
             statuses = ChangesetStatus.query()\
                 .filter(ChangesetStatus.revision.in_(value))\
                 .filter(ChangesetStatus.repo_id == repo_id)\
                 .all()
 
             errors = []
             for status in statuses:
                 if status.pull_request_id:
                     errors.append(['pull_req', status.revision[:12]])
                 elif status.status:
                     errors.append(['status', status.revision[:12]])
 
             if errors:
                 revs = ','.join([x[1] for x in errors])
                 msg = M(self, 'rev_already_reviewed', state, revs=revs)
                 raise formencode.Invalid(
                     msg, value, state, error_dict={'revisions': revs})
 
     return _validator
 
 
 def ValidIp(localizer):
     _ = localizer
 
     class _validator(CIDR):
         messages = {
-            'badFormat': _(u'Please enter a valid IPv4 or IpV6 address'),
+            'badFormat': _('Please enter a valid IPv4 or IpV6 address'),
             'illegalBits': _(
-                u'The network size (bits) must be within the range '
-                u'of 0-32 (not %(bits)r)'),
+                'The network size (bits) must be within the range '
+                'of 0-32 (not %(bits)r)'),
         }
 
         # we ovveride the default to_python() call
         def to_python(self, value, state):
             v = super(_validator, self).to_python(value, state)
             v = safe_unicode(v.strip())
             net = ipaddress.ip_network(address=v, strict=False)
             return str(net)
 
         def validate_python(self, value, state):
             try:
                 addr = safe_unicode(value.strip())
                 # this raises an ValueError if address is not IpV4 or IpV6
                 ipaddress.ip_network(addr, strict=False)
             except ValueError:
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator
 
 
 def FieldKey(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'badFormat': _(
-                u'Key name can only consist of letters, '
-                u'underscore, dash or numbers'),
+                'Key name can only consist of letters, '
+                'underscore, dash or numbers'),
         }
 
         def validate_python(self, value, state):
             if not re.match('[a-zA-Z0-9_-]+$', value):
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator
 
 
 def ValidAuthPlugins(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'import_duplicate': _(
-                u'Plugins %(loaded)s and %(next_to_load)s '
-                u'both export the same name'),
+                'Plugins %(loaded)s and %(next_to_load)s '
+                'both export the same name'),
             'missing_includeme': _(
-                u'The plugin "%(plugin_id)s" is missing an includeme '
-                u'function.'),
+                'The plugin "%(plugin_id)s" is missing an includeme '
+                'function.'),
             'import_error': _(
-                u'Can not load plugin "%(plugin_id)s"'),
+                'Can not load plugin "%(plugin_id)s"'),
             'no_plugin': _(
-                u'No plugin available with ID "%(plugin_id)s"'),
+                'No plugin available with ID "%(plugin_id)s"'),
         }
 
         def _to_python(self, value, state):
             # filter empty values
             return filter(lambda s: s not in [None, ''], value)
 
         def _validate_legacy_plugin_id(self, plugin_id, value, state):
             """
             Validates that the plugin import works. It also checks that the
             plugin has an includeme attribute.
             """
             try:
                 plugin = _import_legacy_plugin(plugin_id)
             except Exception as e:
                 log.exception(
                     'Exception during import of auth legacy plugin "{}"'
                     .format(plugin_id))
                 msg = M(self, 'import_error', state, plugin_id=plugin_id)
                 raise formencode.Invalid(msg, value, state)
 
             if not hasattr(plugin, 'includeme'):
                 msg = M(self, 'missing_includeme', state, plugin_id=plugin_id)
                 raise formencode.Invalid(msg, value, state)
 
             return plugin
 
         def _validate_plugin_id(self, plugin_id, value, state):
             """
             Plugins are already imported during app start up. Therefore this
             validation only retrieves the plugin from the plugin registry and
             if it returns something not None everything is OK.
             """
             plugin = loadplugin(plugin_id)
 
             if plugin is None:
                 msg = M(self, 'no_plugin', state, plugin_id=plugin_id)
                 raise formencode.Invalid(msg, value, state)
 
             return plugin
 
         def validate_python(self, value, state):
             unique_names = {}
             for plugin_id in value:
 
                 # Validate legacy or normal plugin.
                 if plugin_id.startswith(legacy_plugin_prefix):
                     plugin = self._validate_legacy_plugin_id(
                         plugin_id, value, state)
                 else:
                     plugin = self._validate_plugin_id(plugin_id, value, state)
 
                 # Only allow unique plugin names.
                 if plugin.name in unique_names:
                     msg = M(self, 'import_duplicate', state,
                             loaded=unique_names[plugin.name],
                             next_to_load=plugin)
                     raise formencode.Invalid(msg, value, state)
                 unique_names[plugin.name] = plugin
     return _validator
 
 
 def ValidPattern(localizer):
     _ = localizer
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
-            'bad_format': _(u'Url must start with http or /'),
+            'bad_format': _('Url must start with http or /'),
         }
 
         def _to_python(self, value, state):
             patterns = []
 
             prefix = 'new_pattern'
             for name, v in value.items():
                 pattern_name = '_'.join((prefix, 'pattern'))
                 if name.startswith(pattern_name):
                     new_item_id = name[len(pattern_name)+1:]
 
                     def _field(name):
                         return '%s_%s_%s' % (prefix, name, new_item_id)
 
                     values = {
                         'issuetracker_pat': value.get(_field('pattern')),
                         'issuetracker_url': value.get(_field('url')),
                         'issuetracker_pref': value.get(_field('prefix')),
                         'issuetracker_desc': value.get(_field('description'))
                     }
                     new_uid = md5(values['issuetracker_pat'])
 
                     has_required_fields = (
                         values['issuetracker_pat']
                         and values['issuetracker_url'])
 
                     if has_required_fields:
                         # validate url that it starts with http or /
                         # otherwise it can lead to JS injections
                         # e.g specifig javascript:<malicios code>
                         if not values['issuetracker_url'].startswith(('http', '/')):
                             raise formencode.Invalid(
                                 self.message('bad_format', state),
                                 value, state)
 
                         settings = [
                             ('_'.join((key, new_uid)), values[key], 'unicode')
                             for key in values]
                         patterns.append(settings)
 
             value['patterns'] = patterns
             delete_patterns = value.get('uid') or []
             if not isinstance(delete_patterns, (list, tuple)):
                 delete_patterns = [delete_patterns]
             value['delete_patterns'] = delete_patterns
             return value
     return _validator

rhodecode/subscribers.py

@@ -1,398 +1,398 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import io
 import shlex
 
 import math
 import re
 import os
 import datetime
 import logging
 import queue
 import subprocess
 
 
 from dateutil.parser import parse
 from pyramid.threadlocal import get_current_request
 from pyramid.interfaces import IRoutesMapper
 from pyramid.settings import asbool
 from pyramid.path import AssetResolver
 from threading import Thread
 
 from rhodecode.config.jsroutes import generate_jsroutes_content
 from rhodecode.lib.base import get_auth_user
 
 import rhodecode
 
 
 log = logging.getLogger(__name__)
 
 
 def add_renderer_globals(event):
     from rhodecode.lib import helpers
 
     # TODO: When executed in pyramid view context the request is not available
     # in the event. Find a better solution to get the request.
     request = event['request'] or get_current_request()
 
     # Add Pyramid translation as '_' to context
     event['_'] = request.translate
     event['_ungettext'] = request.plularize
     event['h'] = helpers
 
 
 def set_user_lang(event):
     request = event.request
     cur_user = getattr(request, 'user', None)
 
     if cur_user:
         user_lang = cur_user.get_instance().user_data.get('language')
         if user_lang:
             log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
             event.request._LOCALE_ = user_lang
 
 
 def update_celery_conf(event):
     from rhodecode.lib.celerylib.loader import set_celery_conf
     log.debug('Setting celery config from new request')
     set_celery_conf(request=event.request, registry=event.request.registry)
 
 
 def add_request_user_context(event):
     """
     Adds auth user into request context
     """
     request = event.request
     # access req_id as soon as possible
     req_id = request.req_id
 
     if hasattr(request, 'vcs_call'):
         # skip vcs calls
         return
 
     if hasattr(request, 'rpc_method'):
         # skip api calls
         return
 
     auth_user, auth_token = get_auth_user(request)
     request.user = auth_user
     request.user_auth_token = auth_token
     request.environ['rc_auth_user'] = auth_user
     request.environ['rc_auth_user_id'] = auth_user.user_id
     request.environ['rc_req_id'] = req_id
 
 
 def reset_log_bucket(event):
     """
     reset the log bucket on new request
     """
     request = event.request
     request.req_id_records_init()
 
 
 def scan_repositories_if_enabled(event):
     """
     This is subscribed to the `pyramid.events.ApplicationCreated` event. It
     does a repository scan if enabled in the settings.
     """
     settings = event.app.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
     import_on_startup = settings['startup.import_repos']
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
         from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
         repositories = ScmModel().repo_scan(get_rhodecode_base_path())
         repo2db_mapper(repositories, remove_obsolete=False)
 
 
 def write_metadata_if_needed(event):
     """
     Writes upgrade metadata
     """
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     fname = '.rcmetadata.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
     metadata_destination = os.path.join(ini_loc, fname)
 
     def get_update_age():
         now = datetime.datetime.utcnow()
 
         with open(metadata_destination, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return diff.total_seconds() / 60.0
 
         return 0
 
     def write():
         configuration = system_info.SysInfo(
             system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         setup = dict(
             workers=configuration['config']['server:main'].get(
                 'workers', '?'),
             worker_type=configuration['config']['server:main'].get(
                 'worker_class', 'sync'),
         )
         dbinfo = system_info.SysInfo(system_info.database_info)()['value']
         del dbinfo['url']
 
         metadata = dict(
             desc='upgrade metadata info',
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
             platform=system_info.SysInfo(system_info.platform_type)()['value'],
             database=dbinfo,
             cpu=system_info.SysInfo(system_info.cpu)()['value'],
             memory=system_info.SysInfo(system_info.memory)()['value'],
             setup=setup
         )
 
         with open(metadata_destination, 'wb') as f:
             f.write(ext_json.json.dumps(metadata))
 
     settings = event.app.registry.settings
     if settings.get('metadata.skip'):
         return
 
     # only write this every 24h, workers restart caused unwanted delays
     try:
         age_in_min = get_update_age()
     except Exception:
         age_in_min = 0
 
     if age_in_min > 60 * 60 * 24:
         return
 
     try:
         write()
     except Exception:
         pass
 
 
 def write_usage_data(event):
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     settings = event.app.registry.settings
     instance_tag = settings.get('metadata.write_usage_tag')
     if not settings.get('metadata.write_usage'):
         return
 
     def get_update_age(dest_file):
         now = datetime.datetime.utcnow()
 
         with open(dest_file, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return math.ceil(diff.total_seconds() / 60.0)
 
         return 0
 
     utc_date = datetime.datetime.utcnow()
     hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
     fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
         date=utc_date, hour=hour_quarter)
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
 
     usage_dir = os.path.join(ini_loc, '.rcusage')
     if not os.path.isdir(usage_dir):
         os.makedirs(usage_dir)
     usage_metadata_destination = os.path.join(usage_dir, fname)
 
     try:
         age_in_min = get_update_age(usage_metadata_destination)
     except Exception:
         age_in_min = 0
 
     # write every 6th hour
     if age_in_min and age_in_min < 60 * 6:
         log.debug('Usage file created %s minutes ago, skipping (threshold: %s minutes)...',
                   age_in_min, 60 * 6)
         return
 
     def write(dest_file):
         configuration = system_info.SysInfo(system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         metadata = dict(
             desc='Usage data',
             instance_tag=instance_tag,
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
         )
 
         with open(dest_file, 'wb') as f:
             f.write(ext_json.json.dumps(metadata, indent=2, sort_keys=True))
 
     try:
         log.debug('Writing usage file at: %s', usage_metadata_destination)
         write(usage_metadata_destination)
     except Exception:
         pass
 
 
 def write_js_routes_if_enabled(event):
     registry = event.app.registry
 
     mapper = registry.queryUtility(IRoutesMapper)
-    _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
+    _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
 
     def _extract_route_information(route):
         """
         Convert a route into tuple(name, path, args), eg:
             ('show_user', '/profile/%(username)s', ['username'])
         """
 
         routepath = route.pattern
         pattern = route.pattern
 
         def replace(matchobj):
             if matchobj.group(1):
                 return "%%(%s)s" % matchobj.group(1).split(':')[0]
             else:
                 return "%%(%s)s" % matchobj.group(2)
 
         routepath = _argument_prog.sub(replace, routepath)
 
         if not routepath.startswith('/'):
             routepath = '/'+routepath
 
         return (
             route.name,
             routepath,
             [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
               for arg in _argument_prog.findall(pattern)]
         )
 
     def get_routes():
         # pyramid routes
         for route in mapper.get_routes():
             if not route.name.startswith('__'):
                 yield _extract_route_information(route)
 
     if asbool(registry.settings.get('generate_js_files', 'false')):
         static_path = AssetResolver().resolve('rhodecode:public').abspath()
         jsroutes = get_routes()
         jsroutes_file_content = generate_jsroutes_content(jsroutes)
         jsroutes_file_path = os.path.join(
             static_path, 'js', 'rhodecode', 'routes.js')
 
         try:
             with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
                 f.write(jsroutes_file_content)
         except Exception:
             log.exception('Failed to write routes.js into %s', jsroutes_file_path)
 
 
 class Subscriber(object):
     """
     Base class for subscribers to the pyramid event system.
     """
     def __call__(self, event):
         self.run(event)
 
     def run(self, event):
         raise NotImplementedError('Subclass has to implement this.')
 
 
 class AsyncSubscriber(Subscriber):
     """
     Subscriber that handles the execution of events in a separate task to not
     block the execution of the code which triggers the event. It puts the
     received events into a queue from which the worker process takes them in
     order.
     """
     def __init__(self):
         self._stop = False
         self._eventq = queue.Queue()
         self._worker = self.create_worker()
         self._worker.start()
 
     def __call__(self, event):
         self._eventq.put(event)
 
     def create_worker(self):
         worker = Thread(target=self.do_work)
         worker.daemon = True
         return worker
 
     def stop_worker(self):
         self._stop = False
         self._eventq.put(None)
         self._worker.join()
 
     def do_work(self):
         while not self._stop:
             event = self._eventq.get()
             if event is not None:
                 self.run(event)
 
 
 class AsyncSubprocessSubscriber(AsyncSubscriber):
     """
     Subscriber that uses the subprocess module to execute a command if an
     event is received. Events are handled asynchronously::
 
         subscriber = AsyncSubprocessSubscriber('ls -la', timeout=10)
         subscriber(dummyEvent) # running __call__(event)
 
     """
 
     def __init__(self, cmd, timeout=None):
         if not isinstance(cmd, (list, tuple)):
             cmd = shlex.split(cmd)
         super(AsyncSubprocessSubscriber, self).__init__()
         self._cmd = cmd
         self._timeout = timeout
 
     def run(self, event):
         cmd = self._cmd
         timeout = self._timeout
         log.debug('Executing command %s.', cmd)
 
         try:
             output = subprocess.check_output(
                 cmd, timeout=timeout, stderr=subprocess.STDOUT)
             log.debug('Command finished %s', cmd)
             if output:
                 log.debug('Command output: %s', output)
         except subprocess.TimeoutExpired as e:
             log.exception('Timeout while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except subprocess.CalledProcessError as e:
             log.exception('Error while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except Exception:
             log.exception(
                 'Exception while executing command %s.', cmd)

rhodecode/templates/codeblocks/diffs.mako

@@ -1,1404 +1,1404 b''
 <%namespace name="base" file="/base/base.mako"/>
 <%namespace name="commentblock" file="/changeset/changeset_file_comment.mako"/>
 
 <%def name="diff_line_anchor(commit, filename, line, type)"><%
 return '%s_%s_%i' % (h.md5_safe(commit+filename), type, line)
 %></%def>
 
 <%def name="action_class(action)">
 <%
     return {
         '-': 'cb-deletion',
         '+': 'cb-addition',
         ' ': 'cb-context',
         }.get(action, 'cb-empty')
 %>
 </%def>
 
 <%def name="op_class(op_id)">
 <%
     return {
         DEL_FILENODE: 'deletion', # file deleted
         BIN_FILENODE: 'warning' # binary diff hidden
     }.get(op_id, 'addition')
 %>
 </%def>
 
 
 
 <%def name="render_diffset(diffset, commit=None,
 
     # collapse all file diff entries when there are more than this amount of files in the diff
     collapse_when_files_over=20,
 
     # collapse lines in the diff when more than this amount of lines changed in the file diff
     lines_changed_limit=500,
 
     # add a ruler at to the output
     ruler_at_chars=0,
 
     # show inline comments
     use_comments=False,
 
     # disable new comments
     disable_new_comments=False,
 
     # special file-comments that were deleted in previous versions
     # it's used for showing outdated comments for deleted files in a PR
     deleted_files_comments=None,
 
     # for cache purpose
     inline_comments=None,
 
     # additional menu for PRs
     pull_request_menu=None,
 
     # show/hide todo next to comments
     show_todos=True,
 
 )">
 
 <%
     diffset_container_id = h.md5(diffset.target_ref)
     collapse_all = len(diffset.files) > collapse_when_files_over
     active_pattern_entries = h.get_active_pattern_entries(getattr(c, 'repo_name', None))
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE
 %>
 
 %if use_comments:
 
 ## Template for injecting comments
 <div id="cb-comments-inline-container-template" class="js-template">
   ${inline_comments_container([])}
 </div>
 
 <div class="js-template" id="cb-comment-inline-form-template">
     <div class="comment-inline-form ac">
     %if not c.rhodecode_user.is_default:
         ## render template for inline comments
         ${commentblock.comment_form(form_type='inline')}
     %endif
     </div>
 </div>
 
 %endif
 
 %if c.user_session_attrs["diffmode"] == 'sideside':
 <style>
 .wrapper {
     max-width: 1600px !important;
 }
 </style>
 %endif
 
 %if ruler_at_chars:
 <style>
 .diff table.cb .cb-content:after {
     content: "";
     border-left: 1px solid blue;
     position: absolute;
     top: 0;
     height: 18px;
     opacity: .2;
     z-index: 10;
     //## +5 to account for diff action (+/-)
     left: ${ruler_at_chars + 5}ch;
 </style>
 %endif
 
 <div class="diffset ${disable_new_comments and 'diffset-comments-disabled'}">
 
     <div style="height: 20px; line-height: 20px">
         ## expand/collapse action
         <div class="pull-left">
             <a class="${'collapsed' if collapse_all else ''}" href="#expand-files" onclick="toggleExpand(this, '${diffset_container_id}'); return false">
             % if collapse_all:
                 <i class="icon-plus-squared-alt icon-no-margin"></i>${_('Expand all files')}
             % else:
                 <i class="icon-minus-squared-alt icon-no-margin"></i>${_('Collapse all files')}
             % endif
             </a>
 
         </div>
 
         ## todos
         % if show_todos and getattr(c, 'at_version', None):
         <div class="pull-right">
             <i class="icon-flag-filled" style="color: #949494">TODOs:</i>
              ${_('not available in this view')}
         </div>
         % elif show_todos:
         <div class="pull-right">
             <div class="comments-number" style="padding-left: 10px">
                 % if hasattr(c, 'unresolved_comments') and hasattr(c, 'resolved_comments'):
                     <i class="icon-flag-filled" style="color: #949494">TODOs:</i>
                     % if c.unresolved_comments:
                         <a href="#show-todos" onclick="$('#todo-box').toggle(); return false">
                             ${_('{} unresolved').format(len(c.unresolved_comments))}
                         </a>
                     % else:
                         ${_('0 unresolved')}
                     % endif
 
                     ${_('{} Resolved').format(len(c.resolved_comments))}
                 % endif
             </div>
         </div>
         % endif
 
 ##         ## comments
 ##         <div class="pull-right">
 ##             <div class="comments-number" style="padding-left: 10px">
 ##                 % if hasattr(c, 'comments') and hasattr(c, 'inline_cnt'):
 ##                     <i class="icon-comment" style="color: #949494">COMMENTS:</i>
 ##                     % if c.comments:
 ##                         <a href="#comments">${_ungettext("{} General", "{} General", len(c.comments)).format(len(c.comments))}</a>,
 ##                     % else:
 ##                         ${_('0 General')}
 ##                     % endif
 ##
 ##                     % if c.inline_cnt:
 ##                         <a href="#" onclick="return Rhodecode.comments.nextComment();"
 ##                            id="inline-comments-counter">${_ungettext("{} Inline", "{} Inline", c.inline_cnt).format(c.inline_cnt)}
 ##                         </a>
 ##                     % else:
 ##                         ${_('0 Inline')}
 ##                     % endif
 ##                 % endif
 ##
 ##                 % if pull_request_menu:
 ##                     <%
 ##                     outdated_comm_count_ver = pull_request_menu['outdated_comm_count_ver']
 ##                     %>
 ##
 ##                     % if outdated_comm_count_ver:
 ##                         <a href="#" onclick="showOutdated(); Rhodecode.comments.nextOutdatedComment(); return false;">
 ##                             (${_("{} Outdated").format(outdated_comm_count_ver)})
 ##                         </a>
 ##                         <a href="#" class="showOutdatedComments" onclick="showOutdated(this); return false;"> | ${_('show outdated')}</a>
 ##                         <a href="#" class="hideOutdatedComments" style="display: none" onclick="hideOutdated(this); return false;"> | ${_('hide outdated')}</a>
 ##                     % else:
 ##                         (${_("{} Outdated").format(outdated_comm_count_ver)})
 ##                     % endif
 ##
 ##                 % endif
 ##
 ##             </div>
 ##         </div>
 
     </div>
 
     % if diffset.limited_diff:
         <div class="diffset-heading ${(diffset.limited_diff and 'diffset-heading-warning' or '')}">
             <h2 class="clearinner">
                 ${_('The requested changes are too big and content was truncated.')}
                 <a href="${h.current_route_path(request, fulldiff=1)}" onclick="return confirm('${_("Showing a big diff might take some time and resources, continue?")}')">${_('Show full diff')}</a>
             </h2>
         </div>
     % endif
 
     <div id="todo-box">
         % if hasattr(c, 'unresolved_comments') and c.unresolved_comments:
             % for co in c.unresolved_comments:
                 <a class="permalink" href="#comment-${co.comment_id}"
                    onclick="Rhodecode.comments.scrollToComment($('#comment-${co.comment_id}'))">
                     <i class="icon-flag-filled-red"></i>
                     ${co.comment_id}</a>${('' if loop.last else ',')}
             % endfor
         % endif
     </div>
     %if diffset.has_hidden_changes:
         <p class="empty_data">${_('Some changes may be hidden')}</p>
     %elif not diffset.files:
         <p class="empty_data">${_('No files')}</p>
     %endif
 
     <div class="filediffs">
 
     ## initial value could be marked as False later on
     <% over_lines_changed_limit = False %>
     %for i, filediff in enumerate(diffset.files):
 
         %if filediff.source_file_path and filediff.target_file_path:
             %if filediff.source_file_path != filediff.target_file_path:
                  ## file was renamed, or copied
                 %if RENAMED_FILENODE in filediff.patch['stats']['ops']:
                     <%
                         final_file_name = h.literal(u'{} <i class="icon-angle-left"></i> <del>{}</del>'.format(filediff.target_file_path, filediff.source_file_path))
                         final_path = filediff.target_file_path
                     %>
                 %elif COPIED_FILENODE in filediff.patch['stats']['ops']:
                     <%
                         final_file_name = h.literal(u'{} <i class="icon-angle-left"></i> {}'.format(filediff.target_file_path, filediff.source_file_path))
                         final_path = filediff.target_file_path
                     %>
                 %endif
             %else:
                 ## file was modified
                 <%
                     final_file_name = filediff.source_file_path
                     final_path = final_file_name
                 %>
             %endif
         %else:
             %if filediff.source_file_path:
                 ## file was deleted
                 <%
                     final_file_name = filediff.source_file_path
                     final_path =  final_file_name
                 %>
             %else:
                 ## file was added
                 <%
                     final_file_name = filediff.target_file_path
                     final_path = final_file_name
                 %>
             %endif
         %endif
 
         <%
         lines_changed = filediff.patch['stats']['added'] + filediff.patch['stats']['deleted']
         over_lines_changed_limit = lines_changed > lines_changed_limit
         %>
         ## anchor with support of sticky header
         <div class="anchor" id="a_${h.FID(filediff.raw_id, filediff.patch['filename'])}"></div>
 
         <input ${(collapse_all and 'checked' or '')} class="filediff-collapse-state collapse-${diffset_container_id}" id="filediff-collapse-${id(filediff)}" type="checkbox" onchange="updateSticky();">
         <div
             class="filediff"
             data-f-path="${filediff.patch['filename']}"
             data-anchor-id="${h.FID(filediff.raw_id, filediff.patch['filename'])}"
         >
         <label for="filediff-collapse-${id(filediff)}" class="filediff-heading">
             <%
                 file_comments = (get_inline_comments(inline_comments, filediff.patch['filename']) or {}).values()
                 total_file_comments = [_c for _c in h.itertools.chain.from_iterable(file_comments) if not (_c.outdated or _c.draft)]
             %>
             <div class="filediff-collapse-indicator icon-"></div>
 
             ## Comments/Options PILL
             <span class="pill-group pull-right">
                 <span class="pill" op="comments">
                     <i class="icon-comment"></i> ${len(total_file_comments)}
                 </span>
 
                 <details class="details-reset details-inline-block">
                   <summary class="noselect">
                       <i class="pill icon-options cursor-pointer" op="options"></i>
                   </summary>
                   <details-menu class="details-dropdown">
 
                     <div class="dropdown-item">
                         <span>${final_path}</span>
                         <span class="pull-right icon-clipboard clipboard-action" data-clipboard-text="${final_path}" title="Copy file path"></span>
                     </div>
 
                     <div class="dropdown-divider"></div>
 
                    <div class="dropdown-item">
                         <% permalink = request.current_route_url(_anchor='a_{}'.format(h.FID(filediff.raw_id, filediff.patch['filename']))) %>
                         <a href="${permalink}">¶ permalink</a>
                         <span class="pull-right icon-clipboard clipboard-action" data-clipboard-text="${permalink}" title="Copy permalink"></span>
                    </div>
 
 
                   </details-menu>
                 </details>
 
             </span>
 
             ${diff_ops(final_file_name, filediff)}
 
         </label>
 
         ${diff_menu(filediff, use_comments=use_comments)}
         <table id="file-${h.safeid(h.safe_unicode(filediff.patch['filename']))}" data-f-path="${filediff.patch['filename']}" data-anchor-id="${h.FID(filediff.raw_id, filediff.patch['filename'])}" class="code-visible-block cb cb-diff-${c.user_session_attrs["diffmode"]} code-highlight ${(over_lines_changed_limit and 'cb-collapsed' or '')}">
 
         ## new/deleted/empty content case
         % if not filediff.hunks:
             ## Comment container, on "fakes" hunk that contains all data to render comments
             ${render_hunk_lines(filediff, c.user_session_attrs["diffmode"], filediff.hunk_ops, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
         % endif
 
         %if filediff.limited_diff:
                 <tr class="cb-warning cb-collapser">
                     <td class="cb-text" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=6')}>
                         ${_('The requested commit or file is too big and content was truncated.')} <a href="${h.current_route_path(request, fulldiff=1)}" onclick="return confirm('${_("Showing a big diff might take some time and resources, continue?")}')">${_('Show full diff')}</a>
                     </td>
                 </tr>
         %else:
             %if over_lines_changed_limit:
                 <tr class="cb-warning cb-collapser">
                     <td class="cb-text" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=6')}>
                         ${_('This diff has been collapsed as it changes many lines, (%i lines changed)' % lines_changed)}
                         <a href="#" class="cb-expand"
                            onclick="$(this).closest('table').removeClass('cb-collapsed'); updateSticky(); return false;">${_('Show them')}
                         </a>
                         <a href="#" class="cb-collapse"
                            onclick="$(this).closest('table').addClass('cb-collapsed'); updateSticky(); return false;">${_('Hide them')}
                         </a>
                     </td>
                 </tr>
             %endif
         %endif
 
         % for hunk in filediff.hunks:
             <tr class="cb-hunk">
                 <td ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=3' or '')}>
                     ## TODO: dan: add ajax loading of more context here
                     ## <a href="#">
                         <i class="icon-more"></i>
                     ## </a>
                 </td>
                 <td ${(c.user_session_attrs["diffmode"] == 'sideside' and 'colspan=5' or '')}>
                     @@
                     -${hunk.source_start},${hunk.source_length}
                     +${hunk.target_start},${hunk.target_length}
                     ${hunk.section_header}
                 </td>
             </tr>
 
             ${render_hunk_lines(filediff, c.user_session_attrs["diffmode"], hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
         % endfor
 
         <% unmatched_comments = (inline_comments or {}).get(filediff.patch['filename'], {}) %>
 
         ## outdated comments that do not fit into currently displayed lines
         % for lineno, comments in unmatched_comments.items():
 
             %if c.user_session_attrs["diffmode"] == 'unified':
                 % if loop.index == 0:
                 <tr class="cb-hunk">
                     <td colspan="3"></td>
                     <td>
                         <div>
                         ${_('Unmatched/outdated inline comments below')}
                         </div>
                     </td>
                 </tr>
                 % endif
                 <tr class="cb-line">
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                     </td>
                 </tr>
             %elif c.user_session_attrs["diffmode"] == 'sideside':
                 % if loop.index == 0:
                 <tr class="cb-comment-info">
                     <td colspan="2"></td>
                     <td class="cb-line">
                         <div>
                         ${_('Unmatched/outdated inline comments below')}
                         </div>
                     </td>
                     <td colspan="2"></td>
                     <td class="cb-line">
                         <div>
                         ${_('Unmatched/outdated comments below')}
                         </div>
                     </td>
                 </tr>
                 % endif
                 <tr class="cb-line">
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         % if lineno.startswith('o'):
                             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                         % endif
                     </td>
 
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         % if lineno.startswith('n'):
                             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                         % endif
                     </td>
                 </tr>
             %endif
 
         % endfor
 
             </table>
         </div>
     %endfor
 
     ## outdated comments that are made for a file that has been deleted
     % for filename, comments_dict in (deleted_files_comments or {}).items():
 
         <%
             display_state = 'display: none'
             open_comments_in_file = [x for x in comments_dict['comments'] if x.outdated is False]
             if open_comments_in_file:
                 display_state = ''
             fid = str(id(filename))
         %>
         <div class="filediffs filediff-outdated" style="${display_state}">
             <input ${(collapse_all and 'checked' or '')} class="filediff-collapse-state collapse-${diffset_container_id}" id="filediff-collapse-${id(filename)}" type="checkbox" onchange="updateSticky();">
             <div class="filediff" data-f-path="${filename}"  id="a_${h.FID(fid, filename)}">
                 <label for="filediff-collapse-${id(filename)}" class="filediff-heading">
                     <div class="filediff-collapse-indicator icon-"></div>
 
                     <span class="pill">
                         ## file was deleted
                         ${filename}
                     </span>
                     <span class="pill-group pull-left" >
                         ## file op, doesn't need translation
                         <span class="pill" op="removed">unresolved comments</span>
                     </span>
                     <a class="pill filediff-anchor" href="#a_${h.FID(fid, filename)}">¶</a>
                     <span class="pill-group pull-right">
                         <span class="pill" op="deleted">
                             % if comments_dict['stats'] >0:
                                 -${comments_dict['stats']}
                             % else:
                                  ${comments_dict['stats']}
                             % endif
                         </span>
                     </span>
                 </label>
 
                 <table class="cb cb-diff-${c.user_session_attrs["diffmode"]} code-highlight ${(over_lines_changed_limit and 'cb-collapsed' or '')}">
                     <tr>
                         % if c.user_session_attrs["diffmode"] == 'unified':
                         <td></td>
                         %endif
 
                         <td></td>
                         <td class="cb-text cb-${op_class(BIN_FILENODE)}" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=5')}>
                         <strong>${_('This file was removed from diff during updates to this pull-request.')}</strong><br/>
                         ${_('There are still outdated/unresolved comments attached to it.')}
                         </td>
                     </tr>
                     %if c.user_session_attrs["diffmode"] == 'unified':
                     <tr class="cb-line">
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
                             ${inline_comments_container(comments_dict['comments'], active_pattern_entries=active_pattern_entries)}
                         </td>
                     </tr>
                     %elif c.user_session_attrs["diffmode"] == 'sideside':
                     <tr class="cb-line">
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context"></td>
 
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
                             ${inline_comments_container(comments_dict['comments'], active_pattern_entries=active_pattern_entries)}
                         </td>
                     </tr>
                     %endif
                 </table>
             </div>
         </div>
     % endfor
 
 </div>
 </div>
 </%def>
 
 <%def name="diff_ops(file_name, filediff)">
     <%
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE
     %>
     <span class="pill">
         <i class="icon-file-text"></i>
         ${file_name}
     </span>
 
     <span class="pill-group pull-right">
 
         ## ops pills
         %if filediff.limited_diff:
         <span class="pill tooltip" op="limited" title="The stats for this diff are not complete">limited diff</span>
         %endif
 
         %if NEW_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="created">created</span>
             %if filediff['target_mode'].startswith('120'):
         <span class="pill" op="symlink">symlink</span>
             %else:
         <span class="pill" op="mode">${nice_mode(filediff['target_mode'])}</span>
             %endif
         %endif
 
         %if RENAMED_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="renamed">renamed</span>
         %endif
 
         %if COPIED_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="copied">copied</span>
         %endif
 
         %if DEL_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="removed">removed</span>
         %endif
 
         %if CHMOD_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="mode">
             ${nice_mode(filediff['source_mode'])} ➡ ${nice_mode(filediff['target_mode'])}
         </span>
         %endif
 
         %if BIN_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="binary">binary</span>
             %if MOD_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="modified">modified</span>
             %endif
         %endif
 
         <span class="pill" op="added">${('+' if filediff.patch['stats']['added'] else '')}${filediff.patch['stats']['added']}</span>
         <span class="pill" op="deleted">${((h.safe_int(filediff.patch['stats']['deleted']) or 0) * -1)}</span>
 
     </span>
 
 </%def>
 
 <%def name="nice_mode(filemode)">
     ${(filemode.startswith('100') and filemode[3:] or filemode)}
 </%def>
 
 <%def name="diff_menu(filediff, use_comments=False)">
     <div class="filediff-menu">
 
     %if filediff.diffset.source_ref:
 
     ## FILE BEFORE CHANGES
     %if filediff.operation in ['D', 'M']:
         <a
             class="tooltip"
             href="${h.route_path('repo_files',repo_name=filediff.diffset.target_repo_name,commit_id=filediff.diffset.source_ref,f_path=filediff.source_file_path)}"
             title="${h.tooltip(_('Show file at commit: %(commit_id)s') % {'commit_id': filediff.diffset.source_ref[:12]})}"
         >
             ${_('Show file before')}
         </a> |
     %else:
         <span
             class="tooltip"
             title="${h.tooltip(_('File not present at commit: %(commit_id)s') % {'commit_id': filediff.diffset.source_ref[:12]})}"
         >
             ${_('Show file before')}
         </span> |
     %endif
 
     ## FILE AFTER CHANGES
     %if filediff.operation in ['A', 'M']:
         <a
             class="tooltip"
             href="${h.route_path('repo_files',repo_name=filediff.diffset.source_repo_name,commit_id=filediff.diffset.target_ref,f_path=filediff.target_file_path)}"
             title="${h.tooltip(_('Show file at commit: %(commit_id)s') % {'commit_id': filediff.diffset.target_ref[:12]})}"
         >
             ${_('Show file after')}
         </a>
     %else:
         <span
             class="tooltip"
             title="${h.tooltip(_('File not present at commit: %(commit_id)s') % {'commit_id': filediff.diffset.target_ref[:12]})}"
             >
             ${_('Show file after')}
         </span>
     %endif
 
     % if use_comments:
     |
     <a href="#" onclick="Rhodecode.comments.toggleDiffComments(this);return toggleElement(this)"
                 data-toggle-on="${_('Hide comments')}"
                 data-toggle-off="${_('Show comments')}">
         <span class="hide-comment-button">${_('Hide comments')}</span>
     </a>
     % endif
 
     %endif
 
     </div>
 </%def>
 
 
 <%def name="inline_comments_container(comments, active_pattern_entries=None, line_no='', f_path='')">
 
 <div class="inline-comments">
     %for comment in comments:
         ${commentblock.comment_block(comment, inline=True, active_pattern_entries=active_pattern_entries)}
     %endfor
 
     <%
         extra_class = ''
         extra_style = ''
 
         if comments and comments[-1].outdated_at_version(c.at_version_num):
             extra_class = ' comment-outdated'
             extra_style = 'display: none;'
 
     %>
 
     <div class="reply-thread-container-wrapper${extra_class}" style="${extra_style}">
         <div class="reply-thread-container${extra_class}">
             <div class="reply-thread-gravatar">
                 % if c.rhodecode_user.username != h.DEFAULT_USER:
                 ${base.gravatar(c.rhodecode_user.email, 20, tooltip=True, user=c.rhodecode_user)}
                 % endif
             </div>
 
             <div class="reply-thread-reply-button">
                 % if c.rhodecode_user.username != h.DEFAULT_USER:
                 ## initial reply button, some JS logic can append here a FORM to leave a first comment.
                 <button class="cb-comment-add-button" onclick="return Rhodecode.comments.createComment(this, '${f_path}', '${line_no}', null)">Reply...</button>
                 % endif
             </div>
             ##% endif
             <div class="reply-thread-last"></div>
         </div>
     </div>
 </div>
 
 </%def>
 
 <%!
 
 def get_inline_comments(comments, filename):
     if hasattr(filename, 'unicode_path'):
         filename = filename.unicode_path
 
-    if not isinstance(filename, (unicode, str)):
+    if not isinstance(filename, str):
         return None
 
     if comments and filename in comments:
         return comments[filename]
 
     return None
 
 def get_comments_for(diff_type, comments, filename, line_version, line_number):
     if hasattr(filename, 'unicode_path'):
         filename = filename.unicode_path
 
-    if not isinstance(filename, (unicode, str)):
+    if not isinstance(filename, str):
         return None
 
     file_comments = get_inline_comments(comments, filename)
     if file_comments is None:
         return None
 
     line_key = '{}{}'.format(line_version, line_number) ## e.g o37, n12
     if line_key in file_comments:
         data = file_comments.pop(line_key)
         return data
 %>
 
 <%def name="render_hunk_lines_sideside(filediff, hunk, use_comments=False, inline_comments=None, active_pattern_entries=None)">
 
     <% chunk_count = 1 %>
     %for loop_obj, item in h.looper(hunk.sideside):
     <%
     line = item
     i = loop_obj.index
     prev_line = loop_obj.previous
     old_line_anchor, new_line_anchor = None, None
 
     if line.original.lineno:
         old_line_anchor = diff_line_anchor(filediff.raw_id, hunk.source_file_path, line.original.lineno, 'o')
     if line.modified.lineno:
         new_line_anchor = diff_line_anchor(filediff.raw_id, hunk.target_file_path, line.modified.lineno, 'n')
 
     line_action = line.modified.action or line.original.action
     prev_line_action = prev_line and (prev_line.modified.action or prev_line.original.action)
     %>
 
     <tr class="cb-line">
         <td class="cb-data ${action_class(line.original.action)}"
             data-line-no="${line.original.lineno}"
             >
 
             <% line_old_comments, line_old_comments_no_drafts = None, None %>
             %if line.original.get_comment_args:
                 <%
                     line_old_comments = get_comments_for('side-by-side', inline_comments, *line.original.get_comment_args)
                     line_old_comments_no_drafts = [c for c in line_old_comments if not c.draft] if line_old_comments else []
                     has_outdated = any([x.outdated for x in line_old_comments_no_drafts])
                 %>
             %endif
             %if line_old_comments_no_drafts:
                 % if has_outdated:
                     <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(line_old_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
                     <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(line_old_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
         </td>
         <td class="cb-lineno ${action_class(line.original.action)}"
             data-line-no="${line.original.lineno}"
             %if old_line_anchor:
             id="${old_line_anchor}"
             %endif
         >
             %if line.original.lineno:
             <a name="${old_line_anchor}" href="#${old_line_anchor}">${line.original.lineno}</a>
             %endif
         </td>
 
         <% line_no = 'o{}'.format(line.original.lineno) %>
         <td class="cb-content ${action_class(line.original.action)}"
             data-line-no="${line_no}"
             >
             %if use_comments and line.original.lineno:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(line.original.action)}"></span>${line.original.content or '' | n}</span>
 
             %if use_comments and line.original.lineno and line_old_comments:
                 ${inline_comments_container(line_old_comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
 
         </td>
         <td class="cb-data ${action_class(line.modified.action)}"
             data-line-no="${line.modified.lineno}"
             >
             <div>
 
             <% line_new_comments, line_new_comments_no_drafts = None, None %>
             %if line.modified.get_comment_args:
                 <%
                     line_new_comments = get_comments_for('side-by-side', inline_comments, *line.modified.get_comment_args)
                     line_new_comments_no_drafts = [c for c in line_new_comments if not c.draft] if line_new_comments else []
                     has_outdated = any([x.outdated for x in line_new_comments_no_drafts])
                 %>
             %endif
 
             %if line_new_comments_no_drafts:
                 % if has_outdated:
                     <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(line_new_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
                     <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(line_new_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
             </div>
         </td>
         <td class="cb-lineno ${action_class(line.modified.action)}"
             data-line-no="${line.modified.lineno}"
             %if new_line_anchor:
             id="${new_line_anchor}"
             %endif
             >
             %if line.modified.lineno:
                 <a name="${new_line_anchor}" href="#${new_line_anchor}">${line.modified.lineno}</a>
             %endif
         </td>
 
         <% line_no = 'n{}'.format(line.modified.lineno) %>
         <td class="cb-content ${action_class(line.modified.action)}"
             data-line-no="${line_no}"
             >
             %if use_comments and line.modified.lineno:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(line.modified.action)}"></span>${line.modified.content or '' | n}</span>
                 % if line_action in ['+', '-'] and prev_line_action not in ['+', '-']:
                 <div class="nav-chunk" style="visibility: hidden">
                     <i class="icon-eye" title="viewing diff hunk-${hunk.index}-${chunk_count}"></i>
                 </div>
                 <% chunk_count +=1 %>
                 % endif
             %if use_comments and line.modified.lineno and line_new_comments:
             ${inline_comments_container(line_new_comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
 
         </td>
     </tr>
     %endfor
 </%def>
 
 
 <%def name="render_hunk_lines_unified(filediff, hunk, use_comments=False,  inline_comments=None, active_pattern_entries=None)">
     %for old_line_no, new_line_no, action, content, comments_args in hunk.unified:
 
     <%
     old_line_anchor, new_line_anchor = None, None
     if old_line_no:
         old_line_anchor = diff_line_anchor(filediff.raw_id, hunk.source_file_path, old_line_no, 'o')
     if new_line_no:
         new_line_anchor = diff_line_anchor(filediff.raw_id, hunk.target_file_path, new_line_no, 'n')
     %>
     <tr class="cb-line">
         <td class="cb-data ${action_class(action)}">
             <div>
 
                 <% comments, comments_no_drafts = None, None %>
                 %if comments_args:
                     <%
                         comments = get_comments_for('unified', inline_comments, *comments_args)
                         comments_no_drafts = [c for c in line_new_comments if not c.draft] if line_new_comments else []
                         has_outdated = any([x.outdated for x in comments_no_drafts])
                     %>
                 %endif
 
                 % if comments_no_drafts:
                     % if has_outdated:
                         <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                     % else:
                         <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                     % endif
                 % endif
             </div>
         </td>
         <td class="cb-lineno ${action_class(action)}"
             data-line-no="${old_line_no}"
             %if old_line_anchor:
             id="${old_line_anchor}"
             %endif
         >
             %if old_line_anchor:
             <a name="${old_line_anchor}" href="#${old_line_anchor}">${old_line_no}</a>
             %endif
         </td>
         <td class="cb-lineno ${action_class(action)}"
             data-line-no="${new_line_no}"
             %if new_line_anchor:
             id="${new_line_anchor}"
             %endif
         >
             %if new_line_anchor:
             <a name="${new_line_anchor}" href="#${new_line_anchor}">${new_line_no}</a>
             %endif
         </td>
         <% line_no = '{}{}'.format(new_line_no and 'n' or 'o', new_line_no or old_line_no) %>
         <td class="cb-content ${action_class(action)}"
             data-line-no="${line_no}"
             >
             %if use_comments:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(action)}"></span> ${content or '' | n}</span>
             %if use_comments and comments:
             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
         </td>
     </tr>
     %endfor
 </%def>
 
 
 <%def name="render_hunk_lines(filediff, diff_mode, hunk, use_comments, inline_comments, active_pattern_entries)">
     % if diff_mode == 'unified':
         ${render_hunk_lines_unified(filediff, hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
     % elif diff_mode == 'sideside':
         ${render_hunk_lines_sideside(filediff, hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
     % else:
         <tr class="cb-line">
             <td>unknown diff mode</td>
         </tr>
     % endif
 </%def>file changes
 
 
 <%def name="render_add_comment_button(line_no='', f_path='')">
 % if not c.rhodecode_user.is_default:
 <button class="btn btn-small btn-primary cb-comment-box-opener" onclick="return Rhodecode.comments.createComment(this, '${f_path}', '${line_no}', null)">
     <span><i class="icon-comment"></i></span>
 </button>
 % endif
 </%def>
 
 <%def name="render_diffset_menu(diffset, range_diff_on=None, commit=None, pull_request_menu=None)">
     <% diffset_container_id = h.md5(diffset.target_ref) %>
 
     <div id="diff-file-sticky" class="diffset-menu clearinner">
         ## auto adjustable
         <div class="sidebar__inner">
             <div class="sidebar__bar">
             <div class="pull-right">
 
                 <div class="btn-group" style="margin-right: 5px;">
                     <a class="tooltip btn" onclick="scrollDown();return false" title="${_('Scroll to page bottom')}">
                         <i class="icon-arrow_down"></i>
                     </a>
                     <a class="tooltip btn" onclick="scrollUp();return false" title="${_('Scroll to page top')}">
                         <i class="icon-arrow_up"></i>
                     </a>
                 </div>
 
                 <div class="btn-group">
                     <a class="btn tooltip toggle-wide-diff" href="#toggle-wide-diff" onclick="toggleWideDiff(this); return false" title="${h.tooltip(_('Toggle wide diff'))}">
                     <i class="icon-wide-mode"></i>
                     </a>
                 </div>
                 <div class="btn-group">
 
                 <a
                   class="btn ${(c.user_session_attrs["diffmode"] == 'sideside' and 'btn-active')} tooltip"
                   title="${h.tooltip(_('View diff as side by side'))}"
                   href="${h.current_route_path(request, diffmode='sideside')}">
                     <span>${_('Side by Side')}</span>
                 </a>
 
                 <a
                   class="btn ${(c.user_session_attrs["diffmode"] == 'unified' and 'btn-active')} tooltip"
                   title="${h.tooltip(_('View diff as unified'))}" href="${h.current_route_path(request, diffmode='unified')}">
                     <span>${_('Unified')}</span>
                 </a>
 
                 % if range_diff_on is True:
                     <a
                       title="${_('Turn off: Show the diff as commit range')}"
                       class="btn btn-primary"
                       href="${h.current_route_path(request, **{"range-diff":"0"})}">
                         <span>${_('Range Diff')}</span>
                    </a>
                 % elif range_diff_on is False:
                     <a
                       title="${_('Show the diff as commit range')}"
                       class="btn"
                       href="${h.current_route_path(request, **{"range-diff":"1"})}">
                         <span>${_('Range Diff')}</span>
                    </a>
                 % endif
             </div>
                 <div class="btn-group">
 
                 <details class="details-reset details-inline-block">
                   <summary class="noselect btn">
                         <i class="icon-options cursor-pointer" op="options"></i>
                   </summary>
 
                   <div>
                   <details-menu class="details-dropdown" style="top: 35px;">
 
                     <div class="dropdown-item">
                         <div style="padding: 2px 0px">
                         % if request.GET.get('ignorews', '') == '1':
                             <a href="${h.current_route_path(request, ignorews=0)}">${_('Show whitespace changes')}</a>
                         % else:
                             <a href="${h.current_route_path(request, ignorews=1)}">${_('Hide whitespace changes')}</a>
                         % endif
                         </div>
                     </div>
 
                    <div class="dropdown-item">
                        <div style="padding: 2px 0px">
                         % if request.GET.get('fullcontext', '') == '1':
                             <a href="${h.current_route_path(request, fullcontext=0)}">${_('Hide full context diff')}</a>
                         % else:
                             <a href="${h.current_route_path(request, fullcontext=1)}">${_('Show full context diff')}</a>
                         % endif
                         </div>
                    </div>
 
                   </details-menu>
                   </div>
                 </details>
 
                 </div>
         </div>
             <div class="pull-left">
             <div class="btn-group">
               <div class="pull-left">
                 ${h.hidden('file_filter_{}'.format(diffset_container_id))}
               </div>
 
             </div>
             </div>
         </div>
         <div class="fpath-placeholder pull-left">
             <i class="icon-file-text"></i>
             <strong class="fpath-placeholder-text">
             Context file:
             </strong>
         </div>
         <div class="pull-right noselect">
             %if commit:
                 <span>
                     <code>${h.show_id(commit)}</code>
                 </span>
             %elif pull_request_menu and pull_request_menu.get('pull_request'):
                 <span>
                     <code>!${pull_request_menu['pull_request'].pull_request_id}</code>
                 </span>
             %endif
             % if commit or pull_request_menu:
                 <span class="tooltip" title="Navigate to previous or next change inside files." id="diff_nav">Loading diff...:</span>
                 <span class="cursor-pointer" onclick="scrollToPrevChunk(); return false">
                     <i class="icon-angle-up"></i>
                 </span>
                 <span class="cursor-pointer" onclick="scrollToNextChunk(); return false">
                     <i class="icon-angle-down"></i>
                 </span>
             % endif
         </div>
         <div class="sidebar_inner_shadow"></div>
         </div>
     </div>
 
     % if diffset:
         %if diffset.limited_diff:
             <% file_placeholder = _ungettext('%(num)s file changed', '%(num)s files changed', diffset.changed_files) % {'num': diffset.changed_files} %>
         %else:
             <% file_placeholder = h.literal(_ungettext('%(num)s file changed: <span class="op-added">%(linesadd)s inserted</span>, <span class="op-deleted">%(linesdel)s deleted</span>', '%(num)s files changed: <span class="op-added">%(linesadd)s inserted</span>, <span class="op-deleted">%(linesdel)s deleted</span>',
             diffset.changed_files) % {'num': diffset.changed_files, 'linesadd': diffset.lines_added, 'linesdel': diffset.lines_deleted}) %>
 
         %endif
         ## case on range-diff placeholder needs to be updated
         % if range_diff_on is True:
             <% file_placeholder = _('Disabled on range diff') %>
         % endif
 
         <script type="text/javascript">
         var feedFilesOptions = function (query, initialData) {
             var data = {results: []};
             var isQuery = typeof query.term !== 'undefined';
 
             var section = _gettext('Changed files');
             var filteredData = [];
 
             //filter results
             $.each(initialData.results, function (idx, value) {
 
                 if (!isQuery || query.term.length === 0 || value.text.toUpperCase().indexOf(query.term.toUpperCase()) >= 0) {
                     filteredData.push({
                         'id': this.id,
                         'text': this.text,
                         "ops": this.ops,
                     })
                 }
 
             });
 
             data.results = filteredData;
 
             query.callback(data);
         };
 
         var selectionFormatter = function(data, escapeMarkup) {
             var container = '<div class="filelist" style="padding-right:100px">{0}</div>';
             var tmpl = '<div><strong>{0}</strong></div>'.format(escapeMarkup(data['text']));
             var pill = '<div class="pill-group" style="position: absolute; top:7px; right: 0">' +
                         '<span class="pill" op="added">{0}</span>' +
                         '<span class="pill" op="deleted">{1}</span>' +
                        '</div>'
                     ;
             var added = data['ops']['added'];
             if (added === 0) {
                 // don't show +0
                 added = 0;
             } else {
                 added = '+' + added;
             }
 
             var deleted = -1*data['ops']['deleted'];
 
             tmpl += pill.format(added, deleted);
             return container.format(tmpl);
         };
         var formatFileResult = function(result, container, query, escapeMarkup) {
             return selectionFormatter(result, escapeMarkup);
         };
 
         var formatSelection = function (data, container) {
             return '${file_placeholder}'
         };
 
         if (window.preloadFileFilterData === undefined) {
             window.preloadFileFilterData = {}
         }
 
         preloadFileFilterData["${diffset_container_id}"] = {
             results: [
                 % for filediff in diffset.files:
                     {id:"a_${h.FID(filediff.raw_id, filediff.patch['filename'])}",
                      text:"${filediff.patch['filename']}",
                      ops:${h.json.dumps(filediff.patch['stats'])|n}}${('' if loop.last else ',')}
                 % endfor
             ]
         };
 
         var diffFileFilterId = "#file_filter_" + "${diffset_container_id}";
         var diffFileFilter = $(diffFileFilterId).select2({
             'dropdownAutoWidth': true,
             'width': 'auto',
 
             containerCssClass: "drop-menu",
             dropdownCssClass: "drop-menu-dropdown",
             data: preloadFileFilterData["${diffset_container_id}"],
             query: function(query) {
                 feedFilesOptions(query, preloadFileFilterData["${diffset_container_id}"]);
             },
             initSelection: function(element, callback) {
               callback({'init': true});
             },
             formatResult: formatFileResult,
             formatSelection: formatSelection
         });
 
         % if range_diff_on is True:
             diffFileFilter.select2("enable", false);
         % endif
 
         $(diffFileFilterId).on('select2-selecting', function (e) {
             var idSelector = e.choice.id;
 
             // expand the container if we quick-select the field
             $('#'+idSelector).next().prop('checked', false);
             // hide the mast as we later do preventDefault()
             $("#select2-drop-mask").click();
 
             window.location.hash = '#'+idSelector;
             updateSticky();
 
             e.preventDefault();
         });
 
         diffNavText = 'diff navigation:'
 
         getCurrentChunk = function () {
 
             var chunksAll = $('.nav-chunk').filter(function () {
                 return $(this).parents('.filediff').prev().get(0).checked !== true
             })
             var chunkSelected = $('.nav-chunk.selected');
             var initial = false;
 
             if (chunkSelected.length === 0) {
                 // no initial chunk selected, we pick first
                 chunkSelected = $(chunksAll.get(0));
                 var initial = true;
             }
 
             return {
                 'all': chunksAll,
                 'selected': chunkSelected,
                 'initial': initial,
             }
         }
 
         animateDiffNavText = function () {
             var $diffNav = $('#diff_nav')
 
             var callback = function () {
                 $diffNav.animate({'opacity': 1.00}, 200)
             };
             $diffNav.animate({'opacity': 0.15}, 200, callback);
         }
 
         scrollToChunk = function (moveBy) {
             var chunk = getCurrentChunk();
             var all = chunk.all
             var selected = chunk.selected
 
             var curPos = all.index(selected);
             var newPos = curPos;
             if (!chunk.initial) {
                 var newPos = curPos + moveBy;
             }
 
             var curElem = all.get(newPos);
 
             if (curElem === undefined) {
                 // end or back
                 $('#diff_nav').html('no next diff element:')
                 animateDiffNavText()
                 return
             } else if (newPos < 0) {
                 $('#diff_nav').html('no previous diff element:')
                 animateDiffNavText()
                 return
             } else {
                 $('#diff_nav').html(diffNavText)
             }
 
             curElem = $(curElem)
             var offset = 100;
             $(window).scrollTop(curElem.position().top - offset);
 
             //clear selection
             all.removeClass('selected')
             curElem.addClass('selected')
         }
 
         scrollToPrevChunk = function () {
             scrollToChunk(-1)
         }
         scrollToNextChunk = function () {
             scrollToChunk(1)
         }
 
         </script>
     % endif
 
     <script type="text/javascript">
         $('#diff_nav').html('loading diff...') // wait until whole page is loaded
 
         $(document).ready(function () {
 
             var contextPrefix = _gettext('Context file: ');
             ## sticky sidebar
             var sidebarElement = document.getElementById('diff-file-sticky');
             sidebar = new StickySidebar(sidebarElement, {
                   topSpacing: 0,
                   bottomSpacing: 0,
                   innerWrapperSelector: '.sidebar__inner'
             });
             sidebarElement.addEventListener('affixed.static.stickySidebar', function () {
                 // reset our file so it's not holding new value
                 $('.fpath-placeholder-text').html(contextPrefix + ' - ')
             });
 
             updateSticky = function () {
                 sidebar.updateSticky();
                 Waypoint.refreshAll();
             };
 
             var animateText = function (fPath, anchorId) {
                 fPath = Select2.util.escapeMarkup(fPath);
                 $('.fpath-placeholder-text').html(contextPrefix + '<a href="#a_' + anchorId + '">' + fPath + '</a>')
             };
 
             ## dynamic file waypoints
             var setFPathInfo = function(fPath, anchorId){
                 animateText(fPath, anchorId)
             };
 
             var codeBlock = $('.filediff');
 
             // forward waypoint
             codeBlock.waypoint(
                 function(direction) {
                     if (direction === "down"){
                         setFPathInfo($(this.element).data('fPath'), $(this.element).data('anchorId'))
                     }
                 }, {
                     offset: function () {
                         return 70;
                     },
                     context: '.fpath-placeholder'
                 }
             );
 
             // backward waypoint
             codeBlock.waypoint(
                 function(direction) {
                     if (direction === "up"){
                         setFPathInfo($(this.element).data('fPath'), $(this.element).data('anchorId'))
                     }
                 }, {
                     offset: function () {
                         return -this.element.clientHeight + 90;
                     },
                     context: '.fpath-placeholder'
                 }
             );
 
             toggleWideDiff = function (el) {
                 updateSticky();
                 var wide = Rhodecode.comments.toggleWideMode(this);
                 storeUserSessionAttr('rc_user_session_attr.wide_diff_mode', wide);
                 if (wide === true) {
                     $(el).addClass('btn-active');
                 } else {
                     $(el).removeClass('btn-active');
                 }
                 return null;
             };
 
             toggleExpand = function (el, diffsetEl) {
                 var el = $(el);
                 if (el.hasClass('collapsed')) {
                     $('.filediff-collapse-state.collapse-{0}'.format(diffsetEl)).prop('checked', false);
                     el.removeClass('collapsed');
                     el.html(
                         '<i class="icon-minus-squared-alt icon-no-margin"></i>' +
                         _gettext('Collapse all files'));
                 }
                 else {
                     $('.filediff-collapse-state.collapse-{0}'.format(diffsetEl)).prop('checked', true);
                     el.addClass('collapsed');
                     el.html(
                         '<i class="icon-plus-squared-alt icon-no-margin"></i>' +
                         _gettext('Expand all files'));
                 }
                 updateSticky()
             };
 
             toggleCommitExpand = function (el) {
                 var $el = $(el);
                 var commits = $el.data('toggleCommitsCnt');
                 var collapseMsg = _ngettext('Collapse {0} commit', 'Collapse {0} commits', commits).format(commits);
                 var expandMsg = _ngettext('Expand {0} commit', 'Expand {0} commits', commits).format(commits);
 
                 if ($el.hasClass('collapsed')) {
                     $('.compare_select').show();
                     $('.compare_select_hidden').hide();
 
                     $el.removeClass('collapsed');
                     $el.html(
                         '<i class="icon-minus-squared-alt icon-no-margin"></i>' +
                        collapseMsg);
                 }
                 else {
                     $('.compare_select').hide();
                     $('.compare_select_hidden').show();
                     $el.addClass('collapsed');
                     $el.html(
                         '<i class="icon-plus-squared-alt icon-no-margin"></i>' +
                         expandMsg);
                 }
                 updateSticky();
             };
 
             // get stored diff mode and pre-enable it
             if (templateContext.session_attrs.wide_diff_mode === "true") {
                 Rhodecode.comments.toggleWideMode(null);
                 $('.toggle-wide-diff').addClass('btn-active');
                 updateSticky();
             }
 
             // DIFF NAV //
 
             // element to detect scroll direction of
             var $window = $(window);
 
             // initialize last scroll position
             var lastScrollY = $window.scrollTop();
 
             $window.on('resize scrollstop', {latency: 350}, function () {
                 var visibleChunks = $('.nav-chunk').withinviewport({top: 75});
 
                 // get current scroll position
                 var currentScrollY = $window.scrollTop();
 
                 // determine current scroll direction
                 if (currentScrollY > lastScrollY) {
                     var y = 'down'
                 } else if (currentScrollY !== lastScrollY) {
                     var y = 'up';
                 }
 
                 var pos = -1; // by default we use last element in viewport
                 if (y === 'down') {
                     pos = -1;
                 } else if (y === 'up') {
                     pos = 0;
                 }
 
                 if (visibleChunks.length > 0) {
                     $('.nav-chunk').removeClass('selected');
                     $(visibleChunks.get(pos)).addClass('selected');
                 }
 
                 // update last scroll position to current position
                 lastScrollY = currentScrollY;
 
             });
             $('#diff_nav').html(diffNavText);
 
         });
     </script>
 
 </%def>

rhodecode/tests/lib/middleware/test_simplesvn.py

@@ -1,209 +1,208 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-
-from io import StringIO
+import io
 
 import pytest
 from mock import patch, Mock
 
 from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp
 from rhodecode.lib.utils import get_rhodecode_base_path
 
 
 class TestSimpleSvn(object):
     @pytest.fixture(autouse=True)
     def simple_svn(self, baseapp, request_stub):
         base_path = get_rhodecode_base_path()
         self.app = SimpleSvn(
             config={'auth_ret_code': '', 'base_path': base_path},
             registry=request_stub.registry)
 
     def test_get_config(self):
         extras = {'foo': 'FOO', 'bar': 'BAR'}
         config = self.app._create_config(extras, repo_name='test-repo')
         assert config == extras
 
     @pytest.mark.parametrize(
         'method', ['OPTIONS', 'PROPFIND', 'GET', 'REPORT'])
     def test_get_action_returns_pull(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'pull'
 
     @pytest.mark.parametrize(
         'method', [
             'MKACTIVITY', 'PROPPATCH', 'PUT', 'CHECKOUT', 'MKCOL', 'MOVE',
             'COPY', 'DELETE', 'LOCK', 'UNLOCK', 'MERGE'
         ])
     def test_get_action_returns_push(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'push'
 
     @pytest.mark.parametrize(
         'path, expected_name', [
             ('/hello-svn', 'hello-svn'),
             ('/hello-svn/', 'hello-svn'),
             ('/group/hello-svn/', 'group/hello-svn'),
             ('/group/hello-svn/!svn/vcc/default', 'group/hello-svn'),
         ])
     def test_get_repository_name(self, path, expected_name):
         environment = {'PATH_INFO': path}
         name = self.app._get_repository_name(environment)
         assert name == expected_name
 
     def test_get_repository_name_subfolder(self, backend_svn):
         repo = backend_svn.repo
         environment = {
             'PATH_INFO': '/{}/path/with/subfolders'.format(repo.repo_name)}
         name = self.app._get_repository_name(environment)
         assert name == repo.repo_name
 
     def test_create_wsgi_app(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = False
             with patch('rhodecode.lib.middleware.simplesvn.DisabledSimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
     def test_create_wsgi_app_when_enabled(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = True
             with patch('rhodecode.lib.middleware.simplesvn.SimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
 
 class TestSimpleSvnApp(object):
     data = '<xml></xml>'
     path = '/group/my-repo'
-    wsgi_input = StringIO(data)
+    wsgi_input = io.StringIO(data)
     environment = {
         'HTTP_DAV': (
             'http://subversion.tigris.org/xmlns/dav/svn/depth,'
             ' http://subversion.tigris.org/xmlns/dav/svn/mergeinfo'),
         'HTTP_USER_AGENT': 'SVN/1.8.11 (x86_64-linux) serf/1.3.8',
         'REQUEST_METHOD': 'OPTIONS',
         'PATH_INFO': path,
         'wsgi.input': wsgi_input,
         'CONTENT_TYPE': 'text/xml',
         'CONTENT_LENGTH': '130'
     }
 
     def setup_method(self, method):
         self.host = 'http://localhost/'
         base_path = get_rhodecode_base_path()
         self.app = SimpleSvnApp(
             config={'subversion_http_server_url': self.host,
                     'base_path': base_path})
 
     def test_get_request_headers_with_content_type(self):
         expected_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         headers = self.app._get_request_headers(self.environment)
         assert headers == expected_headers
 
     def test_get_request_headers_without_content_type(self):
         environment = self.environment.copy()
         environment.pop('CONTENT_TYPE')
         expected_headers = {
             'Dav': environment['HTTP_DAV'],
             'Content-Length': self.environment['CONTENT_LENGTH'],
             'User-Agent': environment['HTTP_USER_AGENT'],
         }
         request_headers = self.app._get_request_headers(environment)
         assert request_headers == expected_headers
 
     def test_get_response_headers(self):
         headers = {
             'Connection': 'keep-alive',
             'Keep-Alive': 'timeout=5, max=100',
             'Transfer-Encoding': 'chunked',
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         expected_headers = [
             ('MS-Author-Via', 'DAV'),
             ('SVN-Supported-Posts', 'create-txn-with-props'),
         ]
         response_headers = self.app._get_response_headers(headers)
         assert sorted(response_headers) == sorted(expected_headers)
 
     @pytest.mark.parametrize('svn_http_url, path_info, expected_url', [
         ('http://localhost:8200', '/repo_name', 'http://localhost:8200/repo_name'),
         ('http://localhost:8200///', '/repo_name', 'http://localhost:8200/repo_name'),
         ('http://localhost:8200', '/group/repo_name', 'http://localhost:8200/group/repo_name'),
         ('http://localhost:8200/', '/group/repo_name', 'http://localhost:8200/group/repo_name'),
         ('http://localhost:8200/prefix', '/repo_name', 'http://localhost:8200/prefix/repo_name'),
         ('http://localhost:8200/prefix', 'repo_name', 'http://localhost:8200/prefix/repo_name'),
         ('http://localhost:8200/prefix', '/group/repo_name', 'http://localhost:8200/prefix/group/repo_name')
     ])
     def test_get_url(self, svn_http_url, path_info, expected_url):
         url = self.app._get_url(svn_http_url, path_info)
         assert url == expected_url
 
     def test_call(self):
         start_response = Mock()
         response_mock = Mock()
         response_mock.headers = {
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         response_mock.status_code = 200
         response_mock.reason = 'OK'
         with patch('rhodecode.lib.middleware.simplesvn.requests.request') as (
                 request_mock):
             request_mock.return_value = response_mock
             self.app(self.environment, start_response)
 
         expected_url = '{}{}'.format(self.host.strip('/'), self.path)
         expected_request_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         expected_response_headers = [
             ('SVN-Supported-Posts', 'create-txn-with-props'),
             ('MS-Author-Via', 'DAV'),
         ]
         request_mock.assert_called_once_with(
             self.environment['REQUEST_METHOD'], expected_url,
             data=self.data, headers=expected_request_headers, stream=False)
         response_mock.iter_content.assert_called_once_with(chunk_size=1024)
         args, _ = start_response.call_args
         assert args[0] == '200 OK'
         assert sorted(args[1]) == sorted(expected_response_headers)

rhodecode/tests/lib/test_hooks_daemon.py

@@ -1,342 +1,342 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import logging
-from io import StringIO
+import io
 
 import mock
 import pytest
 
 from rhodecode.lib import hooks_daemon
 from rhodecode.tests.utils import assert_message_in_log
 
 
 class TestDummyHooksCallbackDaemon(object):
     def test_hooks_module_path_set_properly(self):
         daemon = hooks_daemon.DummyHooksCallbackDaemon()
         assert daemon.hooks_module == 'rhodecode.lib.hooks_daemon'
 
     def test_logs_entering_the_hook(self):
         daemon = hooks_daemon.DummyHooksCallbackDaemon()
         with mock.patch.object(hooks_daemon.log, 'debug') as log_mock:
             with daemon as return_value:
                 log_mock.assert_called_once_with(
                     'Running `%s` callback daemon', 'DummyHooksCallbackDaemon')
         assert return_value == daemon
 
     def test_logs_exiting_the_hook(self):
         daemon = hooks_daemon.DummyHooksCallbackDaemon()
         with mock.patch.object(hooks_daemon.log, 'debug') as log_mock:
             with daemon:
                 pass
         log_mock.assert_called_with(
             'Exiting `%s` callback daemon', 'DummyHooksCallbackDaemon')
 
 
 class TestHooks(object):
     def test_hooks_can_be_used_as_a_context_processor(self):
         hooks = hooks_daemon.Hooks()
         with hooks as return_value:
             pass
         assert hooks == return_value
 
 
 class TestHooksHttpHandler(object):
     def test_read_request_parses_method_name_and_arguments(self):
         data = {
             'method': 'test',
             'extras': {
                 'param1': 1,
                 'param2': 'a'
             }
         }
         request = self._generate_post_request(data)
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, data['method'], create=True, return_value=1)
 
         with hooks_patcher as hooks_mock:
             MockServer(hooks_daemon.HooksHttpHandler, request)
 
         hooks_mock.assert_called_once_with(data['extras'])
 
     def test_hooks_serialized_result_is_returned(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
         hook_result = {
             'first': 'one',
             'second': 2
         }
         read_patcher = mock.patch.object(
             hooks_daemon.HooksHttpHandler, '_read_request',
             return_value=(rpc_method, {}))
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, rpc_method, create=True,
             return_value=hook_result)
 
         with read_patcher, hooks_patcher:
             server = MockServer(hooks_daemon.HooksHttpHandler, request)
 
         expected_result = json.dumps(hook_result)
         assert server.request.output_stream.buflist[-1] == expected_result
 
     def test_exception_is_returned_in_response(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
         read_patcher = mock.patch.object(
             hooks_daemon.HooksHttpHandler, '_read_request',
             return_value=(rpc_method, {}))
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, rpc_method, create=True,
             side_effect=Exception('Test exception'))
 
         with read_patcher, hooks_patcher:
             server = MockServer(hooks_daemon.HooksHttpHandler, request)
 
         org_exc = json.loads(server.request.output_stream.buflist[-1])
         expected_result = {
             'exception': 'Exception',
             'exception_traceback': org_exc['exception_traceback'],
             'exception_args': ['Test exception']
         }
         assert org_exc == expected_result
 
     def test_log_message_writes_to_debug_log(self, caplog):
         ip_port = ('0.0.0.0', 8888)
         handler = hooks_daemon.HooksHttpHandler(
             MockRequest('POST /'), ip_port, mock.Mock())
         fake_date = '1/Nov/2015 00:00:00'
         date_patcher = mock.patch.object(
             handler, 'log_date_time_string', return_value=fake_date)
         with date_patcher, caplog.at_level(logging.DEBUG):
             handler.log_message('Some message %d, %s', 123, 'string')
 
         expected_message = "HOOKS: {} - - [{}] Some message 123, string".format(ip_port, fake_date)
         assert_message_in_log(
             caplog.records, expected_message,
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def _generate_post_request(self, data):
         payload = json.dumps(data)
         return 'POST / HTTP/1.0\nContent-Length: {}\n\n{}'.format(
             len(payload), payload)
 
 
 class ThreadedHookCallbackDaemon(object):
     def test_constructor_calls_prepare(self):
         prepare_daemon_patcher = mock.patch.object(
             hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
         with prepare_daemon_patcher as prepare_daemon_mock:
             hooks_daemon.ThreadedHookCallbackDaemon()
         prepare_daemon_mock.assert_called_once_with()
 
     def test_run_is_called_on_context_start(self):
         patchers = mock.patch.multiple(
             hooks_daemon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
 
         with patchers as mocks:
             daemon = hooks_daemon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 pass
             mocks['_run'].assert_called_once_with()
         assert daemon_context == daemon
 
     def test_stop_is_called_on_context_exit(self):
         patchers = mock.patch.multiple(
             hooks_daemon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
 
         with patchers as mocks:
             daemon = hooks_daemon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 assert mocks['_stop'].call_count == 0
 
             mocks['_stop'].assert_called_once_with()
         assert daemon_context == daemon
 
 
 class TestHttpHooksCallbackDaemon(object):
     def test_hooks_callback_generates_new_port(self, caplog):
         with caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon.server_address == ('127.0.0.1', 8881)
 
         with caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
         assert daemon._daemon.server_address[1] in range(0, 66000)
         assert daemon._daemon.server_address[0] != '127.0.0.1'
 
     def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon == tcp_server
 
         _, port = tcp_server.server_address
         expected_uri = '{}:{}'.format('127.0.0.1', port)
         msg = 'HOOKS: {}  Preparing HTTP callback daemon registering ' \
               'hook object: rhodecode.lib.hooks_daemon.HooksHttpHandler'.format(expected_uri)
         assert_message_in_log(
             caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_prepare_inits_hooks_uri_and_logs_it(
             self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
 
         _, port = tcp_server.server_address
         expected_uri = '{}:{}'.format('127.0.0.1', port)
         assert daemon.hooks_uri == expected_uri
 
         msg = 'HOOKS: {}  Preparing HTTP callback daemon registering ' \
               'hook object: rhodecode.lib.hooks_daemon.HooksHttpHandler'.format(expected_uri)
         assert_message_in_log(
             caplog.records, msg,
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_run_creates_a_thread(self, tcp_server):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread) as thread_mock:
             daemon._run()
 
         thread_mock.assert_called_once_with(
             target=tcp_server.serve_forever,
             kwargs={'poll_interval': daemon.POLL_INTERVAL})
         assert thread.daemon is True
         thread.start.assert_called_once_with()
 
     def test_run_logs(self, tcp_server, caplog):
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
             daemon._run()
 
         assert_message_in_log(
             caplog.records,
             'Running event loop of callback daemon in background thread',
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
             with daemon:
                 assert daemon._daemon == tcp_server
                 assert daemon._callback_thread == thread
 
         assert daemon._daemon is None
         assert daemon._callback_thread is None
         tcp_server.shutdown.assert_called_with()
         thread.join.assert_called_once_with()
 
         assert_message_in_log(
             caplog.records, 'Waiting for background thread to finish.',
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def _tcp_patcher(self, tcp_server):
         return mock.patch.object(
             hooks_daemon, 'TCPServer', return_value=tcp_server)
 
     def _thread_patcher(self, thread):
         return mock.patch.object(
             hooks_daemon.threading, 'Thread', return_value=thread)
 
 
 class TestPrepareHooksDaemon(object):
     @pytest.mark.parametrize('protocol', ('http',))
     def test_returns_dummy_hooks_callback_daemon_when_using_direct_calls(
             self, protocol):
         expected_extras = {'extra1': 'value1'}
         callback, extras = hooks_daemon.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol,
             host='127.0.0.1', use_direct_calls=True)
         assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
         expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
         expected_extras['time'] = extras['time']
         assert 'extra1' in extras
 
     @pytest.mark.parametrize('protocol, expected_class', (
         ('http', hooks_daemon.HttpHooksCallbackDaemon),
     ))
     def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
             self, protocol, expected_class):
         expected_extras = {
             'extra1': 'value1',
             'txn_id': 'txnid2',
             'hooks_protocol': protocol.lower()
         }
         callback, extras = hooks_daemon.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
             use_direct_calls=False,
             txn_id='txnid2')
         assert isinstance(callback, expected_class)
         extras.pop('hooks_uri')
         expected_extras['time'] = extras['time']
         assert extras == expected_extras
 
     @pytest.mark.parametrize('protocol', (
         'invalid',
         'Http',
         'HTTP',
     ))
     def test_raises_on_invalid_protocol(self, protocol):
         expected_extras = {
             'extra1': 'value1',
             'hooks_protocol': protocol.lower()
         }
         with pytest.raises(Exception):
             callback, extras = hooks_daemon.prepare_callback_daemon(
                 expected_extras.copy(),
                 protocol=protocol, host='127.0.0.1',
                 use_direct_calls=False)
 
 
 class MockRequest(object):
     def __init__(self, request):
         self.request = request
-        self.input_stream = StringIO(b'{}'.format(self.request))
-        self.output_stream = StringIO()
+        self.input_stream = io.StringIO(b'{}'.format(self.request))
+        self.output_stream = io.StringIO()
 
     def makefile(self, mode, *args, **kwargs):
         return self.output_stream if mode == 'wb' else self.input_stream
 
 
 class MockServer(object):
     def __init__(self, handler_cls, request):
         ip_port = ('0.0.0.0', 8888)
         self.request = MockRequest(request)
         self.server_address = ip_port
         self.handler = handler_cls(self.request, ip_port, self)
 
 
 @pytest.fixture()
 def tcp_server():
     server = mock.Mock()
     server.server_address = ('127.0.0.1', 8881)
     return server

rhodecode/tests/vcs/test_archives.py

@@ -1,176 +1,176 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import datetime
 import os
 import shutil
 import tarfile
 import tempfile
 import zipfile
-from io import StringIO
+import io
 
 import mock
 import pytest
 
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestArchives(BackendTestMixin):
 
     @pytest.fixture(autouse=True)
     def tempfile(self, request):
         self.temp_file = tempfile.mkstemp()[1]
 
         @request.addfinalizer
         def cleanup():
             os.remove(self.temp_file)
 
     @classmethod
     def _get_commits(cls):
         start_date = datetime.datetime(2010, 1, 1, 20)
         yield {
             'message': 'Initial Commit',
             'author': 'Joe Doe <[e-mail]>',
             'date': start_date + datetime.timedelta(hours=12),
             'added': [
                 FileNode('executable_0o100755', '...', mode=0o100755),
                 FileNode('executable_0o100500', '...', mode=0o100500),
                 FileNode('not_executable', '...', mode=0o100644),
             ],
         }
         for x in range(5):
             yield {
                 'message': 'Commit %d' % x,
                 'author': 'Joe Doe <[e-mail]>',
                 'date': start_date + datetime.timedelta(hours=12 * x),
                 'added': [
                     FileNode('%d/file_%d.txt' % (x, x), content='Foobar %d' % x),
                 ],
             }
 
     @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
     def test_archive_tar(self, compressor):
         self.tip.archive_repo(
             self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
         out_dir = tempfile.mkdtemp()
         out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
         out_file.extractall(out_dir)
         out_file.close()
 
         for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
             with open(os.path.join(out_dir, 'repo/' + node_path)) as f:
                 file_content = f.read()
             assert file_content == self.tip.get_node(node_path).content
 
         shutil.rmtree(out_dir)
 
     @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
     def test_archive_tar_symlink(self, compressor):
         return False
 
     @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
     def test_archive_tar_file_modes(self, compressor):
         self.tip.archive_repo(
             self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
         out_dir = tempfile.mkdtemp()
         out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
         out_file.extractall(out_dir)
         out_file.close()
         dest = lambda inp: os.path.join(out_dir, 'repo/' + inp)
 
         assert oct(os.stat(dest('not_executable')).st_mode) == '0100644'
 
     def test_archive_zip(self):
         self.tip.archive_repo(self.temp_file, kind='zip', archive_dir_name='repo')
         out = zipfile.ZipFile(self.temp_file)
 
         for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
-            decompressed = StringIO.StringIO()
+            decompressed = io.StringIO()
             decompressed.write(out.read('repo/' + node_path))
             assert decompressed.getvalue() == \
                 self.tip.get_node(node_path).content
             decompressed.close()
 
     def test_archive_zip_with_metadata(self):
         self.tip.archive_repo(self.temp_file, kind='zip',
                               archive_dir_name='repo', write_metadata=True)
 
         out = zipfile.ZipFile(self.temp_file)
         metafile = out.read('repo/.archival.txt')
 
         raw_id = self.tip.raw_id
         assert 'commit_id:%s' % raw_id in metafile
 
         for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
-            decompressed = StringIO.StringIO()
+            decompressed = io.StringIO()
             decompressed.write(out.read('repo/' + node_path))
             assert decompressed.getvalue() == \
                 self.tip.get_node(node_path).content
             decompressed.close()
 
     def test_archive_wrong_kind(self):
         with pytest.raises(ImproperArchiveTypeError):
             self.tip.archive_repo(self.temp_file, kind='wrong kind')
 
 
 @pytest.fixture()
 def base_commit():
     """
     Prepare a `base.BaseCommit` just enough for `_validate_archive_prefix`.
     """
     commit = base.BaseCommit()
     commit.repository = mock.Mock()
     commit.repository.name = u'fake_repo'
     commit.short_id = 'fake_id'
     return commit
 
 
 @pytest.mark.parametrize("prefix", [u"unicode-prefix", u"Ünïcödë"])
 def test_validate_archive_prefix_enforces_bytes_as_prefix(prefix, base_commit):
     with pytest.raises(ValueError):
         base_commit._validate_archive_prefix(prefix)
 
 
 def test_validate_archive_prefix_empty_prefix(base_commit):
     # TODO: johbo: Should raise a ValueError here.
     with pytest.raises(VCSError):
         base_commit._validate_archive_prefix('')
 
 
 def test_validate_archive_prefix_with_leading_slash(base_commit):
     # TODO: johbo: Should raise a ValueError here.
     with pytest.raises(VCSError):
         base_commit._validate_archive_prefix('/any')
 
 
 def test_validate_archive_prefix_falls_back_to_repository_name(base_commit):
     prefix = base_commit._validate_archive_prefix(None)
     expected_prefix = base_commit._ARCHIVE_PREFIX_TEMPLATE.format(
         repo_name='fake_repo',
         short_id='fake_id')
     assert isinstance(prefix, str)
     assert prefix == expected_prefix