python3: fixed various code issues

- iterators
- use of u''

rhodecode/api/__init__.py

@@ -1,578 +1,578 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import sys
 import types
 import fnmatch
 
 import decorator
 import venusian
 from collections import OrderedDict
 
 from pyramid.exceptions import ConfigurationError
 from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPNotFound
 
 from rhodecode.api.exc import (
     JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
 DEFAULT_URL = '/_admin/apiv2'
 
 
 def find_methods(jsonrpc_methods, pattern):
     matches = OrderedDict()
     if not isinstance(pattern, (list, tuple)):
         pattern = [pattern]
 
     for single_pattern in pattern:
         for method_name, method in jsonrpc_methods.items():
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
 
 
 class ExtJsonRenderer(object):
     """
     Custom renderer that mkaes use of our ext_json lib
 
     """
 
     def __init__(self, serializer=json.dumps, **kw):
         """ Any keyword arguments will be passed to the ``serializer``
         function."""
         self.serializer = serializer
         self.kw = kw
 
     def __call__(self, info):
         """ Returns a plain JSON-encoded string with content-type
         ``application/json``. The content-type may be overridden by
         setting ``request.response.content_type``."""
 
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
 
             return self.serializer(value, **self.kw)
 
         return _render
 
 
 def jsonrpc_response(request, result):
     rpc_id = getattr(request, 'rpc_id', None)
     response = request.response
 
     # store content_type before render is called
     ct = response.content_type
 
     ret_value = ''
     if rpc_id:
         ret_value = {
             'id': rpc_id,
             'result': result,
             'error': None,
         }
 
         # fetch deprecation warnings, and store it inside results
         deprecation = getattr(request, 'rpc_deprecation', None)
         if deprecation:
             ret_value['DEPRECATION_WARNING'] = deprecation
 
     raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
     response.body = safe_str(raw_body, response.charset)
 
     if ct == response.default_content_type:
         response.content_type = 'application/json'
 
     return response
 
 
 def jsonrpc_error(request, message, retid=None, code=None, headers=None):
     """
     Generate a Response object with a JSON-RPC error body
 
     :param code:
     :param retid:
     :param message:
     """
     err_dict = {'id': retid, 'result': None, 'error': message}
     body = render(DEFAULT_RENDERER, err_dict, request=request).encode('utf-8')
 
     return Response(
         body=body,
         status=code,
         content_type='application/json',
         headerlist=headers
     )
 
 
 def exception_view(exc, request):
     rpc_id = getattr(request, 'rpc_id', None)
 
     if isinstance(exc, JSONRPCError):
         fault_message = safe_str(exc.message)
         log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCValidationError):
         colander_exc = exc.colander_exception
         # TODO(marcink): think maybe of nicer way to serialize errors ?
         fault_message = colander_exc.asdict()
         log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCForbidden):
         fault_message = 'Access was denied to this resource.'
         log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, HTTPNotFound):
         method = request.rpc_method
         log.debug('json-rpc method `%s` not found in list of '
                   'api calls: %s, rpc_id:%s',
                   method, request.registry.jsonrpc_methods.keys(), rpc_id)
 
         similar = 'none'
         try:
             similar_paterns = ['*{}*'.format(x) for x in method.split('_')]
             similar_found = find_methods(
                 request.registry.jsonrpc_methods, similar_paterns)
             similar = ', '.join(similar_found.keys()) or similar
         except Exception:
             # make the whole above block safe
             pass
 
         fault_message = "No such method: {}. Similar methods: {}".format(
             method, similar)
     else:
         fault_message = 'undefined error'
         exc_info = exc.exc_info()
         store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = "{}.{}".format(exc.__class__.__module__, exc.__class__.__name__)
             statsd.incr('rhodecode_exception_total',
                         tags=["exc_source:api", "type:{}".format(exc_type)])
 
     return jsonrpc_error(request, fault_message, rpc_id)
 
 
 def request_view(request):
     """
     Main request handling method. It handles all logic to call a specific
     exposed method
     """
     # cython compatible inspect
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
         api_user = User.get_by_auth_token(request.rpc_api_key)
 
         if api_user is None:
             return jsonrpc_error(
                 request, retid=request.rpc_id, message='Invalid API KEY')
 
         if not api_user.active:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='Request from this user not allowed')
 
         # check if we are allowed to use this IP
         auth_u = AuthUser(
             api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
         if not auth_u.ip_allowed:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='Request from IP:%s not allowed' % (
                     request.rpc_ip_addr,))
         else:
             log.info('Access for IP:%s allowed', request.rpc_ip_addr)
 
         # register our auth-user
         request.rpc_user = auth_u
         request.environ['rc_auth_user_id'] = auth_u.user_id
 
         # now check if token is valid for API
         auth_token = request.rpc_api_key
         token_match = api_user.authenticate_by_token(
             auth_token, roles=[UserApiKeys.ROLE_API])
         invalid_token = not token_match
 
         log.debug('Checking if API KEY is valid with proper role')
         if invalid_token:
             return jsonrpc_error(
                 request, retid=request.rpc_id,
                 message='API KEY invalid or, has bad role for an API call')
 
     except Exception:
         log.exception('Error on API AUTH')
         return jsonrpc_error(
             request, retid=request.rpc_id, message='Invalid API KEY')
 
     method = request.rpc_method
     func = request.registry.jsonrpc_methods[method]
 
     # now that we have a method, add request._req_params to
     # self.kargs and dispatch control to WGIController
     argspec = inspect.getargspec(func)
     arglist = argspec[0]
     defaults = map(type, argspec[3] or [])
     default_empty = types.NotImplementedType
 
     # kw arguments required by this method
-    func_kwargs = dict(itertools.izip_longest(
+    func_kwargs = dict(itertools.zip_longest(
         reversed(arglist), reversed(defaults), fillvalue=default_empty))
 
     # This attribute will need to be first param of a method that uses
     # api_key, which is translated to instance of user at that name
     user_var = 'apiuser'
     request_var = 'request'
 
     for arg in [user_var, request_var]:
         if arg not in arglist:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message='This method [%s] does not support '
                         'required parameter `%s`' % (func.__name__, arg))
 
     # get our arglist and check if we provided them as args
     for arg, default in func_kwargs.items():
         if arg in [user_var, request_var]:
             # user_var and request_var are pre-hardcoded parameters and we
             # don't need to do any translation
             continue
 
         # skip the required param check if it's default value is
         # NotImplementedType (default_empty)
         if default == default_empty and arg not in request.rpc_params:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message=('Missing non optional `%s` arg in JSON DATA' % arg)
             )
 
     # sanitize extra passed arguments
     for k in request.rpc_params.keys()[:]:
         if k not in func_kwargs:
             del request.rpc_params[k]
 
     call_params = request.rpc_params
     call_params.update({
         'request': request,
         'apiuser': auth_u
     })
 
     # register some common functions for usage
     attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
 
     statsd = request.registry.statsd
 
     try:
         ret_value = func(**call_params)
         resp = jsonrpc_response(request, ret_value)
         if statsd:
             statsd.incr('rhodecode_api_call_success_total')
         return resp
     except JSONRPCBaseError:
         raise
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
         exc_id, exc_type_name = store_exception(
             id(exc_info), exc_info, prefix='rhodecode-api')
         error_headers = [('RhodeCode-Exception-Id', str(exc_id)),
                          ('RhodeCode-Exception-Type', str(exc_type_name))]
         err_resp = jsonrpc_error(
             request, retid=request.rpc_id, message='Internal server error',
             headers=error_headers)
         if statsd:
             statsd.incr('rhodecode_api_call_fail_total')
         return err_resp
 
 
 def setup_request(request):
     """
     Parse a JSON-RPC request body. It's used inside the predicates method
     to validate and bootstrap requests for usage in rpc calls.
 
     We need to raise JSONRPCError here if we want to return some errors back to
     user.
     """
 
     log.debug('Executing setup request: %r', request)
     request.rpc_ip_addr = get_ip_addr(request.environ)
     # TODO(marcink): deprecate GET at some point
     if request.method not in ['POST', 'GET']:
         log.debug('unsupported request method "%s"', request.method)
         raise JSONRPCError(
             'unsupported request method "%s". Please use POST' % request.method)
 
     if 'CONTENT_LENGTH' not in request.environ:
         log.debug("No Content-Length")
         raise JSONRPCError("Empty body, No Content-Length in request")
 
     else:
         length = request.environ['CONTENT_LENGTH']
         log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
     log.debug("Loading JSON body now")
     try:
         json_body = json.loads(raw_body)
     except ValueError as e:
         # catch JSON errors Here
         raise JSONRPCError("JSON parse error ERR:%s RAW:%r" % (e, raw_body))
 
     request.rpc_id = json_body.get('id')
     request.rpc_method = json_body.get('method')
 
     # check required base parameters
     try:
         api_key = json_body.get('api_key')
         if not api_key:
             api_key = json_body.get('auth_token')
 
         if not api_key:
             raise KeyError('api_key or auth_token')
 
         # TODO(marcink): support passing in token in request header
 
         request.rpc_api_key = api_key
         request.rpc_id = json_body['id']
         request.rpc_method = json_body['method']
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
         log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError('Incorrect JSON data. Missing %s' % e)
 
     log.debug('setup complete, now handling method:%s rpcid:%s',
               request.rpc_method, request.rpc_id, )
 
 
 class RoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return 'jsonrpc route = %s' % self.val
 
     phash = text
 
     def __call__(self, info, request):
         if self.val:
             # potentially setup and bootstrap our call
             setup_request(request)
 
             # Always return True so that even if it isn't a valid RPC it
             # will fall through to the underlaying handlers like notfound_view
             return True
 
 
 class NotFoundPredicate(object):
     def __init__(self, val, config):
         self.val = val
         self.methods = config.registry.jsonrpc_methods
 
     def text(self):
         return 'jsonrpc method not found = {}.'.format(self.val)
 
     phash = text
 
     def __call__(self, info, request):
         return hasattr(request, 'rpc_method')
 
 
 class MethodPredicate(object):
     def __init__(self, val, config):
         self.method = val
 
     def text(self):
         return 'jsonrpc method = %s' % self.method
 
     phash = text
 
     def __call__(self, context, request):
         # we need to explicitly return False here, so pyramid doesn't try to
         # execute our view directly. We need our main handler to execute things
         return getattr(request, 'rpc_method') == self.method
 
 
 def add_jsonrpc_method(config, view, **kwargs):
     # pop the method name
     method = kwargs.pop('method', None)
 
     if method is None:
         raise ConfigurationError(
             'Cannot register a JSON-RPC method without specifying the "method"')
 
     # we define custom predicate, to enable to detect conflicting methods,
     # those predicates are kind of "translation" from the decorator variables
     # to internal predicates names
 
     kwargs['jsonrpc_method'] = method
 
     # register our view into global view store for validation
     config.registry.jsonrpc_methods[method] = view
 
     # we're using our main request_view handler, here, so each method
     # has a unified handler for itself
     config.add_view(request_view, route_name='apiv2', **kwargs)
 
 
 class jsonrpc_method(object):
     """
     decorator that works similar to @add_view_config decorator,
     but tailored for our JSON RPC
     """
 
     venusian = venusian  # for testing injection
 
     def __init__(self, method=None, **kwargs):
         self.method = method
         self.kwargs = kwargs
 
     def __call__(self, wrapped):
         kwargs = self.kwargs.copy()
         kwargs['method'] = self.method or wrapped.__name__
         depth = kwargs.pop('_depth', 0)
 
         def callback(context, name, ob):
             config = context.config.with_package(info.module)
             config.add_jsonrpc_method(view=ob, **kwargs)
 
         info = venusian.attach(wrapped, callback, category='pyramid',
                                depth=depth + 1)
         if info.scope == 'class':
             # ensure that attr is set if decorating a class method
             kwargs.setdefault('attr', wrapped.__name__)
 
         kwargs['_info'] = info.codeinfo  # fbo action_method
         return wrapped
 
 
 class jsonrpc_deprecated_method(object):
     """
     Marks method as deprecated, adds log.warning, and inject special key to
     the request variable to mark method as deprecated.
     Also injects special docstring that extract_docs will catch to mark
     method as deprecated.
 
     :param use_method: specify which method should be used instead of
         the decorated one
 
     Use like::
 
         @jsonrpc_method()
         @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
         def old_func(request, apiuser, arg1, arg2):
             ...
     """
 
     def __init__(self, use_method, deprecated_at_version):
         self.use_method = use_method
         self.deprecated_at_version = deprecated_at_version
         self.deprecated_msg = ''
 
     def __call__(self, func):
         self.deprecated_msg = 'Please use method `{method}` instead.'.format(
             method=self.use_method)
 
         docstring = """\n
         .. deprecated:: {version}
 
            {deprecation_message}
 
         {original_docstring}
         """
         func.__doc__ = docstring.format(
             version=self.deprecated_at_version,
             deprecation_message=self.deprecated_msg,
             original_docstring=func.__doc__)
         return decorator.decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         log.warning('DEPRECATED API CALL on function %s, please '
                     'use `%s` instead', func, self.use_method)
         # alter function docstring to mark as deprecated, this is picked up
         # via fabric file that generates API DOC.
         result = func(*fargs, **fkwargs)
 
         request = fargs[0]
         request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
         return result
 
 
 def add_api_methods(config):
     from rhodecode.api.views import (
         deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
         server_api, search_api, testing_api, user_api, user_group_api)
 
     config.scan('rhodecode.api.views')
 
 
 def includeme(config):
     plugin_module = 'rhodecode.api'
     plugin_settings = get_plugin_settings(
         plugin_module, config.registry.settings)
 
     if not hasattr(config.registry, 'jsonrpc_methods'):
         config.registry.jsonrpc_methods = OrderedDict()
 
     # match filter by given method only
     config.add_view_predicate('jsonrpc_method', MethodPredicate)
     config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
 
     config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer(
         serializer=json.dumps, indent=4))
     config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
 
     config.add_route_predicate(
         'jsonrpc_call', RoutePredicate)
 
     config.add_route(
         'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
 
     # register some exception handling view
     config.add_view(exception_view, context=JSONRPCBaseError)
     config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
 
     add_api_methods(config)

rhodecode/api/views/server_api.py

@@ -1,419 +1,419 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import itertools
 import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_repo_store(request, apiuser):
     """
     Returns the |RCE| repository storage information.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     path = VcsSettingsModel().get_repos_location()
     return {"path": path}
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 
 @jsonrpc_method()
 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
     """
     Triggers a session cleanup action.
 
     If the ``older_then`` option is set, only sessions that hasn't been
     accessed in the given number of days will be removed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param older_then: Deletes session that hasn't been accessed
         in given number of days.
     :type older_then: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result: {
         "backend": "<type of backend>",
         "sessions_removed": <number_of_removed_sessions>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during session cleanup'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     older_then = safe_int(Optional.extract(older_then)) or 60
     older_than_seconds = 60 * 60 * 24 * older_then
 
     config = system_info.rhodecode_config().get_value()['value']['config']
     session_model = user_sessions.get_session_handler(
         config.get('beaker.session.type', 'memory'))(config)
 
     backend = session_model.SESSION_TYPE
     try:
         cleaned = session_model.clean_sessions(
             older_than_seconds=older_than_seconds)
         return {'sessions_removed': cleaned, 'backend': backend}
     except user_sessions.CleanupCommand as msg:
         return {'cleanup_command': msg.message, 'backend': backend}
     except Exception as e:
         log.exception('Failed session cleanup')
         raise JSONRPCError(
             'Error occurred during session cleanup'
         )
 
 
 @jsonrpc_method()
 def get_method(request, apiuser, pattern=Optional('*')):
     """
     Returns list of all available API methods. By default match pattern
     os "*" but any other pattern can be specified. eg *comment* will return
     all methods with comment inside them. If just single method is matched
     returned data will also include method specification
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param pattern: pattern to match method names against
     :type pattern: Optional("*")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "changeset_comment",
         "comment_pull_request",
         "comment_commit"
       ]
       error :  null
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "comment_commit",
         {
           "apiuser": "<RequiredType>",
           "comment_type": "<Optional:u'note'>",
           "commit_id": "<RequiredType>",
           "message": "<RequiredType>",
           "repoid": "<RequiredType>",
           "request": "<RequiredType>",
           "resolves_comment_id": "<Optional:None>",
           "status": "<Optional:None>",
           "userid": "<Optional:<OptionalAttr:apiuser>>"
         }
       ]
       error :  null
     """
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     pattern = Optional.extract(pattern)
 
     matches = find_methods(request.registry.jsonrpc_methods, pattern)
 
     args_desc = []
     if len(matches) == 1:
         func = matches[matches.keys()[0]]
 
         argspec = inspect.getargspec(func)
         arglist = argspec[0]
         defaults = map(repr, argspec[3] or [])
 
         default_empty = '<RequiredType>'
 
         # kw arguments required by this method
-        func_kwargs = dict(itertools.izip_longest(
+        func_kwargs = dict(itertools.zip_longest(
             reversed(arglist), reversed(defaults), fillvalue=default_empty))
         args_desc.append(func_kwargs)
 
     return matches.keys() + args_desc
 
 
 @jsonrpc_method()
 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
     """
     Stores sent exception inside the built-in exception tracker in |RCE| server.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     :param exc_data_json: JSON data with exception e.g
         {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
     :type exc_data_json: JSON data
 
     :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
     :type prefix: Optional("rhodecode")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": {
         "exc_id": 139718459226384,
         "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
       }
       error :  null
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     prefix = Optional.extract(prefix)
     exc_id = exc_tracking.generate_id()
 
     try:
         exc_data = json.loads(exc_data_json)
     except Exception:
         log.error('Failed to parse JSON: %r', exc_data_json)
         raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
                            'Please make sure it contains a valid JSON.')
 
     try:
         exc_traceback = exc_data['exc_traceback']
         exc_type_name = exc_data['exc_type_name']
     except KeyError as err:
         raise JSONRPCError('Missing exc_traceback, or exc_type_name '
                            'in exc_data_json field. Missing: {}'.format(err))
 
     exc_tracking._store_exception(
         exc_id=exc_id, exc_traceback=exc_traceback,
         exc_type_name=exc_type_name, prefix=prefix)
 
     exc_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=exc_id)
     return {'exc_id': exc_id, 'exc_url': exc_url}

rhodecode/apps/admin/views/permissions.py

@@ -1,479 +1,479 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import logging
 import formencode
 import formencode.htmlfill
 import datetime
 from pyramid.interfaces import IRoutesMapper
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
 from rhodecode import events
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.utils2 import aslist, safe_unicode
 from rhodecode.model.db import (
     or_, coalesce, User, UserIpMap, UserSshKeys)
 from rhodecode.model.forms import (
     ApplicationPermissionsForm, ObjectPermissionsForm, UserPermissionsForm)
 from rhodecode.model.meta import Session
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminPermissionsView(BaseAppView, DataGridAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=self.request.translate)
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_application(self):
         c = self.load_default_context()
         c.active = 'application'
 
         c.user = User.get_default_user(refresh=True)
 
         app_settings = c.rc_config
 
         defaults = {
             'anonymous': c.user.active,
             'default_register_message': app_settings.get(
                 'rhodecode_register_message')
         }
         defaults.update(c.user.get_default_perms())
 
         data = render('rhodecode:templates/admin/permissions/permissions.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_application_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'application'
 
         _form = ApplicationPermissionsForm(
             self.request.translate,
             [x[0] for x in c.register_choices],
             [x[0] for x in c.password_reset_choices],
             [x[0] for x in c.extern_activate_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_application_permissions(form_result)
 
             settings = [
                 ('register_message', 'default_register_message'),
             ]
             for setting, form_key in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key])
                 Session().add(sett)
 
             Session().commit()
             h.flash(_('Application permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_application'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_objects(self):
         c = self.load_default_context()
         c.active = 'objects'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_objects_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'objects'
 
         _form = ObjectPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_perms_choices],
             [x[0] for x in c.group_perms_choices],
             [x[0] for x in c.user_group_perms_choices],
         )()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_object_permissions(form_result)
 
             Session().commit()
             h.flash(_('Object permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_object'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_branch(self):
         c = self.load_default_context()
         c.active = 'branch'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_global(self):
         c = self.load_default_context()
         c.active = 'global'
 
         c.user = User.get_default_user(refresh=True)
         defaults = {}
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/permissions/permissions.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def permissions_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
 
         _form = UserPermissionsForm(
             self.request.translate,
             [x[0] for x in c.repo_create_choices],
             [x[0] for x in c.repo_create_on_write_choices],
             [x[0] for x in c.repo_group_create_choices],
             [x[0] for x in c.user_group_create_choices],
             [x[0] for x in c.fork_choices],
             [x[0] for x in c.inherit_default_permission_choices])()
 
         try:
             form_result = _form.to_python(dict(self.request.POST))
             form_result.update({'perm_user_name': User.DEFAULT_USER})
             PermissionModel().update_user_permissions(form_result)
 
             Session().commit()
             h.flash(_('Global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
 
             data = render(
                 'rhodecode:templates/admin/permissions/permissions.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during update of permissions")
             h.flash(_('Error occurred during update of permissions'),
                     category='error')
 
         affected_user_ids = [User.get_default_user_id()]
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('admin_permissions_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_ips(self):
         c = self.load_default_context()
         c.active = 'ips'
 
         c.user = User.get_default_user(refresh=True)
         c.user_ip_map = (
             UserIpMap.query().filter(UserIpMap.user == c.user).all())
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def permissions_overview(self):
         c = self.load_default_context()
         c.active = 'perms'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_token_access(self):
         from rhodecode import CONFIG
 
         c = self.load_default_context()
         c.active = 'auth_token_access'
 
         c.user = User.get_default_user(refresh=True)
         c.perm_user = c.user.AuthUser()
 
         mapper = self.request.registry.queryUtility(IRoutesMapper)
         c.view_data = []
 
-        _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
+        _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
         introspector = self.request.registry.introspector
 
         view_intr = {}
         for view_data in introspector.get_category('views'):
             intr = view_data['introspectable']
 
             if 'route_name' in intr and intr['attr']:
                 view_intr[intr['route_name']] = '{}:{}'.format(
                     str(intr['derived_callable'].__name__), intr['attr']
                 )
 
         c.whitelist_key = 'api_access_controllers_whitelist'
         c.whitelist_file = CONFIG.get('__file__')
         whitelist_views = aslist(
             CONFIG.get(c.whitelist_key), sep=',')
 
         for route_info in mapper.get_routes():
             if not route_info.name.startswith('__'):
                 routepath = route_info.pattern
 
                 def replace(matchobj):
                     if matchobj.group(1):
                         return "{%s}" % matchobj.group(1).split(':')[0]
                     else:
                         return "{%s}" % matchobj.group(2)
 
                 routepath = _argument_prog.sub(replace, routepath)
 
                 if not routepath.startswith('/'):
                     routepath = '/' + routepath
 
                 view_fqn = view_intr.get(route_info.name, 'NOT AVAILABLE')
                 active = view_fqn in whitelist_views
                 c.view_data.append((route_info.name, view_fqn, routepath, active))
 
         c.whitelist_views = whitelist_views
         return self._get_template_context(c)
 
     def ssh_enabled(self):
         return self.request.registry.settings.get(
             'ssh.generate_authorized_keyfile')
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys(self):
         c = self.load_default_context()
         c.active = 'ssh_keys'
         c.ssh_enabled = self.ssh_enabled()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys_data(self):
         _ = self.request.translate
         self.load_default_context()
         column_map = {
             'fingerprint': 'ssh_key_fingerprint',
             'username': User.username
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
 
         ssh_keys_data_total_count = UserSshKeys.query()\
             .count()
 
         # json generate
         base_q = UserSshKeys.query().join(UserSshKeys.user)
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 UserSshKeys.ssh_key_fingerprint.ilike(like_expression),
             ))
 
         users_data_total_filtered_count = base_q.count()
 
         sort_col = self._get_order_col(order_by, UserSshKeys)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['created_on']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         ssh_keys = base_q.all()
 
         ssh_keys_data = []
         for ssh_key in ssh_keys:
             ssh_keys_data.append({
                 "username": h.gravatar_with_user(self.request, ssh_key.user.username),
                 "fingerprint": ssh_key.ssh_key_fingerprint,
                 "description": ssh_key.description,
                 "created_on": h.format_date(ssh_key.created_on),
                 "accessed_on": h.format_date(ssh_key.accessed_on),
                 "action": h.link_to(
                     _('Edit'), h.route_path('edit_user_ssh_keys',
                                             user_id=ssh_key.user.user_id))
             })
 
         data = ({
             'draw': draw,
             'data': ssh_keys_data,
             'recordsTotal': ssh_keys_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
         })
 
         return data
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_update(self):
         _ = self.request.translate
         self.load_default_context()
 
         ssh_enabled = self.ssh_enabled()
         key_file = self.request.registry.settings.get(
             'ssh.authorized_keys_file_path')
         if ssh_enabled:
             events.trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_('Updated SSH keys file: {}').format(key_file),
                     category='success')
         else:
             h.flash(_('SSH key support is disabled in .ini file'),
                     category='warning')
 
         raise HTTPFound(h.route_path('admin_permissions_ssh_keys'))

rhodecode/apps/file_store/utils.py

@@ -1,58 +1,57 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-
+import io
 import uuid
-from io import StringIO
 import pathlib2
 
 
 def get_file_storage(settings):
     from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
     from rhodecode.apps.file_store import config_keys
     store_path = settings.get(config_keys.store_path)
     return LocalFileStorage(base_path=store_path)
 
 
 def splitext(filename):
     ext = ''.join(pathlib2.Path(filename).suffixes)
     return filename, ext
 
 
 def uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
 
     :param filename: the original filename
     :param randomized: define if filename should be stable (sha1 based) or randomized
     """
 
     _, ext = splitext(filename)
     if randomized:
         uid = uuid.uuid4()
     else:
         hash_key = '{}.{}'.format(filename, 'store')
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
 
 
 def bytes_to_file_obj(bytes_data):
-    return StringIO.StringIO(bytes_data)
+    return io.StringIO(bytes_data)

rhodecode/apps/login/tests/test_login.py

@@ -1,580 +1,580 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urllib.parse
 
 import mock
 import pytest
 
 from rhodecode.tests import (
     assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
     no_newline_id_generator)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.lib.auth import check_password
 from rhodecode.lib import helpers as h
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import User, Notification, UserApiKeys
 from rhodecode.model.meta import Session
 
 fixture = Fixture()
 
 whitelist_view = ['RepoCommitsView:repo_commit_raw']
 
 
 def route_path(name, params=None, **kwargs):
     import urllib.request, urllib.parse, urllib.error
     from rhodecode.apps._base import ADMIN_PREFIX
 
     base_url = {
         'login': ADMIN_PREFIX + '/login',
         'logout': ADMIN_PREFIX + '/logout',
         'register': ADMIN_PREFIX + '/register',
         'reset_password':
             ADMIN_PREFIX + '/password_reset',
         'reset_password_confirmation':
             ADMIN_PREFIX + '/password_reset_confirmation',
 
         'admin_permissions_application':
             ADMIN_PREFIX + '/permissions/application',
         'admin_permissions_application_update':
             ADMIN_PREFIX + '/permissions/application/update',
 
         'repo_commit_raw': '/{repo_name}/raw-changeset/{commit_id}'
 
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.parse.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app')
 class TestLoginController(object):
     destroy_users = set()
 
     @classmethod
     def teardown_class(cls):
         fixture.destroy_users(cls.destroy_users)
 
     def teardown_method(self, method):
         for n in Notification.query().all():
             Session().delete(n)
 
         Session().commit()
         assert Notification.query().all() == []
 
     def test_index(self):
         response = self.app.get(route_path('login'))
         assert response.status == '200 OK'
         # Test response...
 
     def test_login_admin_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'test12'}, status=302)
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_admin'
         response.mustcontain('logout')
 
     def test_login_regular_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_regular',
                                   'password': 'test12'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_regular'
         response.mustcontain('logout')
 
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.auth_restriction(self.app._pyramid_registry,
                                       RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_regular_forbidden_when_scope_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.scope_restriction(self.app._pyramid_registry,
                                        RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_ok_came_from(self):
         test_came_from = '/_admin/users?branch=stable'
         _url = '{}?came_from={}'.format(route_path('login'), test_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
 
         assert 'branch=stable' in response.location
         response = response.follow()
 
         assert response.status == '200 OK'
         response.mustcontain('Users administration')
 
     def test_redirect_to_login_with_get_args(self):
         with fixture.anon_access(False):
             kwargs = {'branch': 'stable'}
             response = self.app.get(
                 h.route_path('repo_summary', repo_name=HG_REPO, _query=kwargs),
                 status=302)
 
-            response_query = urllib.parse.urlparse.parse_qsl(response.location)
+            response_query = urllib.parse.parse_qsl(response.location)
             assert 'branch=stable' in response_query[0][1]
 
     def test_login_form_with_get_args(self):
         _url = '{}?came_from=/_admin/users,branch=stable'.format(route_path('login'))
         response = self.app.get(_url)
         assert 'branch%3Dstable' in response.form.action
 
     @pytest.mark.parametrize("url_came_from", [
         'data:text/html,<script>window.alert("xss")</script>',
         'mailto:[e-mail]',
         'file:///etc/passwd',
         'ftp://some.ftp.server',
         'http://other.domain',
         '/\r\nX-Forwarded-Host: http://example.org',
     ], ids=no_newline_id_generator)
     def test_login_bad_came_froms(self, url_came_from):
         _url = '{}?came_from={}'.format(route_path('login'), url_came_from)
         response = self.app.post(
             _url,
             {'username': 'test_admin', 'password': 'test12'})
         assert response.status == '302 Found'
         response = response.follow()
         assert response.status == '200 OK'
         assert response.request.path == '/'
 
     def test_login_short_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'as'})
         assert response.status == '200 OK'
 
         response.mustcontain('Enter 3 characters or more')
 
     def test_login_wrong_non_ascii_password(self, user_regular):
         response = self.app.post(
             route_path('login'),
             {'username': user_regular.username,
              'password': u'invalid-non-asci\xe4'.encode('utf8')})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_with_non_ascii_password(self, user_util):
         password = u'valid-non-ascii\xe4'
         user = user_util.create_user(password=password)
         response = self.app.post(
             route_path('login'),
             {'username': user.username,
              'password': password})
         assert response.status_code == 302
 
     def test_login_wrong_username_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'error',
                                   'password': 'test12'})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_admin_ok_password_migration(self, real_crypto_backend):
         from rhodecode.lib import auth
 
         # create new user, with sha256 password
         temp_user = 'test_admin_sha256'
         user = fixture.create_user(temp_user)
         user.password = auth._RhodeCodeCryptoSha256().hash_create(
             b'test123')
         Session().add(user)
         Session().commit()
         self.destroy_users.add(temp_user)
         response = self.app.post(route_path('login'),
                                  {'username': temp_user,
                                   'password': 'test123'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == temp_user
         response.mustcontain('logout')
 
         # new password should be bcrypted, after log-in and transfer
         user = User.get_by_username(temp_user)
         assert user.password.startswith('$')
 
     # REGISTRATIONS
     def test_register(self):
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
     def test_register_err_same_username(self):
         uname = 'test_admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': uname,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = 'Username "%(username)s" already exists'
         msg = msg % {'username': uname}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_err_same_email(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_0',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_same_email_case_sensitive(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_1',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_wrong_data(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': 'test',
                 'password_confirmation': 'test',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assert response.status == '200 OK'
         response.mustcontain('An email address must contain a single @')
         response.mustcontain('Enter a value 6 characters long or more')
 
     def test_register_err_username(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'error user',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         response.mustcontain('An email address must contain a single @')
         response.mustcontain(
             'Username may only contain '
             'alphanumeric characters underscores, '
             'periods or dashes and must begin with '
             'alphanumeric character')
 
     def test_register_err_case_sensitive(self):
         usr = 'Test_Admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': usr,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'Username "%(username)s" already exists'
         msg = msg % {'username': usr}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_special_chars(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xxxaxn',
                 'password': 'ąćźżąśśśś',
                 'password_confirmation': 'ąćźżąśśśś',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         msg = u'Invalid characters (non-ascii) in password'
         response.mustcontain(msg)
 
     def test_register_password_mismatch(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': '123qwe',
                 'password_confirmation': 'qwe123',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         msg = u'Passwords do not match'
         response.mustcontain(msg)
 
     def test_register_ok(self):
         username = 'test_regular4'
         password = 'qweqwe'
         email = '[e-mail]'
         name = 'testname'
         lastname = 'testlastname'
 
         # this initializes a session
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
 
         response = self.app.post(
             route_path('register'),
             {
                 'username': username,
                 'password': password,
                 'password_confirmation': password,
                 'email': email,
                 'firstname': name,
                 'lastname': lastname,
                 'admin': True
             },
             status=302
         )  # This should be overridden
 
         assert_session_flash(
             response, 'You have successfully registered with RhodeCode. You can log-in now.')
 
         ret = Session().query(User).filter(
             User.username == 'test_regular4').one()
         assert ret.username == username
         assert check_password(password, ret.password)
         assert ret.email == email
         assert ret.name == name
         assert ret.lastname == lastname
         assert ret.auth_tokens is not None
         assert not ret.admin
 
     def test_forgot_password_wrong_mail(self):
         bad_email = '[e-mail]'
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         response = self.app.post(
             route_path('reset_password'), {'email': bad_email, }
         )
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
     def test_forgot_password(self, user_util):
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         user = user_util.create_user()
         user_id = user.user_id
         email = user.email
 
         response = self.app.post(route_path('reset_password'), {'email': email, })
 
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
         # BAD KEY
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), 'badkey')
         response = self.app.get(confirm_url, status=302)
         assert response.location.endswith(route_path('reset_password'))
         assert_session_flash(response, 'Given reset token is invalid')
 
         response.follow()  # cleanup flash
 
         # GOOD KEY
         key = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == user_id)\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_PASSWORD_RESET)\
             .first()
 
         assert key
 
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), key.api_key)
         response = self.app.get(confirm_url)
         assert response.status == '302 Found'
         assert response.location.endswith(route_path('login'))
 
         assert_session_flash(
             response,
             'Your password reset was successful, '
             'a new password has been sent to your email')
 
         response.follow()
 
     def _get_api_whitelist(self, values=None):
         config = {'api_access_controllers_whitelist': values or []}
         return config
 
     @pytest.mark.parametrize("test_name, auth_token", [
         ('none', None),
         ('empty_string', ''),
         ('fake_number', '123456'),
         ('proper_auth_token', None)
     ])
     def test_access_not_whitelisted_page_via_auth_token(
             self, test_name, auth_token, user_admin):
 
         whitelist = self._get_api_whitelist([])
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert [] == whitelist['api_access_controllers_whitelist']
             if test_name == 'proper_auth_token':
                 # use builtin if api_key is None
                 auth_token = user_admin.api_key
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=302)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('none', None, 302),
         ('empty_string', '', 302),
         ('fake_number', '123456', 302),
         ('proper_auth_token', None, 200)
     ])
     def test_access_whitelisted_page_via_auth_token(
             self, test_name, auth_token, code, user_admin):
 
         whitelist = self._get_api_whitelist(whitelist_view)
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == whitelist['api_access_controllers_whitelist']
 
             if test_name == 'proper_auth_token':
                 auth_token = user_admin.api_key
                 assert auth_token
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('proper_auth_token', None, 200),
         ('wrong_auth_token', '123456', 302),
     ])
     def test_access_whitelisted_page_via_auth_token_bound_to_token(
             self, test_name, auth_token, code, user_admin):
 
         expected_token = auth_token
         if test_name == 'proper_auth_token':
             auth_token = user_admin.api_key
             expected_token = auth_token
             assert auth_token
 
         whitelist = self._get_api_whitelist([
             'RepoCommitsView:repo_commit_raw@{}'.format(expected_token)])
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     def test_access_page_via_extra_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=200)
 
     def test_access_page_via_expired_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             # patch the api key and make it expired
             new_auth_token.expires = 0
             Session().add(new_auth_token)
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=302)

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1877 +1,1877 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest, HTTPConflict)
 
 from pyramid.renderers import render
 
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.exceptions import CommentVersionMismatch
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry
 from rhodecode.lib.vcs.backends.base import (
     EmptyCommit, UpdateFailureReason, unicode_to_reference)
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     func, false, or_, PullRequest, ChangesetComment, ChangesetStatus, Repository,
     PullRequestReviewers)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
         # backward compat., we use for OLD PRs a plain renderer
         c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, search_q=search_q, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments_count = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr,
                 include_drafts=False, count_only=True)
 
             review_statuses = pr.reviewers_statuses(user=self._rhodecode_db_user)
             my_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
             if review_statuses and review_statuses[4]:
                 _review_obj, _user, _reasons, _mandatory, statuses = review_statuses
                 my_review_status = statuses[0][1].status
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name,
                                 short=True),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'my_status': _render('pullrequest_status',
                                      my_review_status),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on),
                                       pr.versions_count),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', comments_count),
                 'comments_raw': comments_count,
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      ancestor_commit,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff, hide_whitespace_changes, diff_context, use_ancestor=True):
 
         target_commit_final = target_commit
         source_commit_final = source_commit
 
         if use_ancestor:
             # we might want to not use it for versions
             target_ref_id = ancestor_commit.raw_id
             target_commit_final = ancestor_commit
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit_final),
             target_node_getter=codeblocks.diffset_node_getter(source_commit_final),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_ref_id, source_ref_id)
 
         return diffset
 
     def _get_range_diffset(self, source_scm, source_repo,
                            commit1, commit2, diff_limit, file_limit,
                            fulldiff, hide_whitespace_changes, diff_context):
         vcs_diff = source_scm.get_diff(
             commit1, commit2,
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
             source_node_getter=codeblocks.diffset_node_getter(commit1),
             target_node_getter=codeblocks.diffset_node_getter(commit2))
 
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
         return diffset
 
     def register_comments_vars(self, c, pull_request, versions, include_drafts=True):
         comments_model = CommentsModel()
 
         # GENERAL COMMENTS with versions #
         q = comments_model._all_general_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         general_comments = q
 
         # pick comments we want to render at current version
         c.comment_versions = comments_model.aggregate_comments(
             general_comments, versions, c.at_version_num)
 
         # INLINE COMMENTS with versions  #
         q = comments_model._all_inline_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         inline_comments = q
 
         c.inline_versions = comments_model.aggregate_comments(
             inline_comments, versions, c.at_version_num, inline=True)
 
         # Comments inline+general
         if c.at_version:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['display']
             c.comments = c.comment_versions[c.at_version_num]['display']
         else:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['until']
             c.comments = c.comment_versions[c.at_version_num]['until']
 
         return general_comments, inline_comments
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_show(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         c.state_progressing = pull_request.is_state_changing()
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         _new_state = {
             'created': PullRequest.STATE_CREATED,
         }.get(self.request.GET.get('force_state'))
         can_force_state = c.is_super_admin or HasRepoPermissionAny('repository.admin')(c.repo_name)
 
         if can_force_state and _new_state:
             with pull_request.set_state(PullRequest.STATE_UPDATING, final_state=_new_state):
                 h.flash(
                     _('Pull Request state was force changed to `{}`').format(_new_state),
                     category='success')
                 Session().commit()
 
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
         force_refresh = str2bool(self.request.GET.get('force_refresh'))
         c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         # fetch global flags of ignore ws or context lines
         diff_context = diffs.get_diff_context(self.request)
         hide_whitespace_changes = diffs.get_diff_whitespace_flag(self.request)
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
 
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
             # not allow to browse versions for closed PR
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
 
         c.commit_versions = PullRequestModel().pr_commits_versions(versions)
 
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.at_version_index = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
                               if prev_at_version and prev_at_version != PullRequest.LATEST_VER
                               else None)
         c.from_version_index = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
             log.warning('Mismatch between the current repo: %s, and target %s',
                         self.db_repo_name, pull_request_at_ver.target_repo.repo_name)
             raise HTTPNotFound()
 
         c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
         c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
         # inject latest version
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         if compare or (at_version and not at_version == PullRequest.LATEST_VER):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
                 c.forbid_adding_reviewers = rules.get('forbid_adding_reviewers')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, auth_user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
 
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
         c.pr_merge_source_commit = _merge_check.source_commit
         c.pr_merge_target_commit = _merge_check.target_commit
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
         c.reviewers_count = pull_request.reviewers_count
         c.observers_count = pull_request.observers_count
 
         # reviewers and statuses
         c.pull_request_default_reviewers_data_json = json.dumps(pull_request.reviewer_data)
         c.pull_request_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
         c.pull_request_set_observers_data_json = collections.OrderedDict({'observers': []})
 
         for review_obj, member, reasons, mandatory, status in pull_request_at_ver.reviewers_statuses():
             member_reviewer = h.reviewer_as_json(
                 member, reasons=reasons, mandatory=mandatory,
                 role=review_obj.role,
                 user_group=review_obj.rule_user_group_data()
             )
 
             current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
             member_reviewer['review_status'] = current_review_status
             member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
             member_reviewer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
         c.pull_request_set_reviewers_data_json = json.dumps(c.pull_request_set_reviewers_data_json)
 
         for observer_obj, member in pull_request_at_ver.observers():
             member_observer = h.reviewer_as_json(
                 member, reasons=[], mandatory=False,
                 role=observer_obj.role,
                 user_group=observer_obj.rule_user_group_data()
             )
             member_observer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_observers_data_json['observers'].append(member_observer)
 
         c.pull_request_set_observers_data_json = json.dumps(c.pull_request_set_observers_data_json)
 
         general_comments, inline_comments = \
             self.register_comments_vars(c, pull_request_latest, versions)
 
         # TODOs
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request_latest)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request_latest)
 
         # Drafts
         c.draft_comments = CommentsModel().get_pull_request_drafts(
             self._rhodecode_db_user.user_id,
             pull_request_latest)
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm
         if shadow_scm:
             commits_source_repo = shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
         version_normalized = version or PullRequest.LATEST_VER
         from_version_normalized = from_version or PullRequest.LATEST_VER
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = self.get_recache_flag()
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
 
         if not force_recache and not c.range_diff_on and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             # NOTE(marcink): we reach potentially unreachable errors when a PR has
             # merge errors resulting in potentially hidden commits in the shadow repo.
             maybe_unreachable = _merge_check.MERGE_CHECK in _merge_check.error_details \
                                 and _merge_check.merge_response
             maybe_unreachable = maybe_unreachable \
                                 and _merge_check.merge_response.metadata.get('unresolved_files')
             log.debug("Using unreachable commits due to MERGE_CHECK in merge simulation")
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm,
                 maybe_unreachable=maybe_unreachable)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             use_ancestor = True
             if from_version_normalized != version_normalized:
                 use_ancestor = False
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
             else:
                 try:
                     c.diffset = self._get_diffset(
                         c.source_repo.repo_name, commits_source_repo,
                         c.ancestor_commit,
                         source_ref_id, target_ref_id,
                         target_commit, source_commit,
                         diff_limit, file_limit, c.fulldiff,
                         hide_whitespace_changes, diff_context,
                         use_ancestor=use_ancestor
                     )
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, c.diffset, diff_commit_cache)
                 except CommitDoesNotExistError:
                     log.exception('Failed to generate diffset')
                     c.missing_commits = True
 
             if not c.missing_commits:
 
                 c.limited_diff = c.diffset.limited_diff
 
                 # calculate removed files that are bound to comments
                 comment_deleted_files = [
                     fname for fname in display_inline_comments
                     if fname not in c.diffset.file_stats]
 
                 c.deleted_files_comments = collections.defaultdict(dict)
                 for fname, per_line_comments in display_inline_comments.items():
                     if fname in comment_deleted_files:
                         c.deleted_files_comments[fname]['stats'] = 0
                         c.deleted_files_comments[fname]['comments'] = list()
                         for lno, comments in per_line_comments.items():
                             c.deleted_files_comments[fname]['comments'].extend(comments)
 
                 # maybe calculate the range diff
                 if c.range_diff_on:
                     # TODO(marcink): set whitespace/context
                     context_lcl = 3
                     ign_whitespace_lcl = False
 
                     for commit in c.commit_ranges:
                         commit2 = commit
                         commit1 = commit.first_parent
 
                         range_diff_cache_file_path = diff_cache_exist(
                             cache_path, 'diff', commit.raw_id,
                             ign_whitespace_lcl, context_lcl, c.fulldiff)
 
                         cached_diff = None
                         if caching_enabled:
                             cached_diff = load_cached_diff(range_diff_cache_file_path)
 
                         has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                         if not force_recache and has_proper_diff_cache:
                             diffset = cached_diff['diff']
                         else:
                             diffset = self._get_range_diffset(
                                 commits_source_repo, source_repo,
                                 commit1, commit2, diff_limit, file_limit,
                                 c.fulldiff, ign_whitespace_lcl, context_lcl
                             )
 
                         # save cached diff
                         if caching_enabled:
                             cache_diff(range_diff_cache_file_path, diffset, None)
 
                         c.changes[commit.raw_id] = diffset
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
 
             def mark(cs, fw):
-                return list(h.itertools.izip_longest([], cs, fillvalue=fw))
+                return list(h.itertools.zip_longest([], cs, fillvalue=fw))
 
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
             is_reviewer = PullRequestModel().is_user_reviewer(
                 pull_request, self._rhodecode_user)
             if is_reviewer:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm,
             maybe_unreachable=False):
 
         commit_cache = collections.OrderedDict()
         missing_requirements = False
 
         try:
             pre_load = ["author", "date", "message", "branch", "parents"]
 
             pull_request_commits = pull_request_at_ver.revisions
             log.debug('Loading %s commits from %s',
                       len(pull_request_commits), commits_source_repo)
 
             for rev in pull_request_commits:
                 comm = commits_source_repo.get_commit(commit_id=rev, pre_load=pre_load,
                                                       maybe_unreachable=maybe_unreachable)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         except CommitDoesNotExistError:
             log.warning('Failed to get commit from `{}` repo'.format(
                 commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning('Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
 
         pr_ancestor_id = pull_request_at_ver.common_ancestor_id
 
         try:
             ancestor_commit = source_scm.get_commit(pr_ancestor_id)
         except Exception:
             ancestor_commit = None
 
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent and c.has_origin_repo_read_perm:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
             'source_refs_json': json.dumps(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
             'target_refs_json': json.dumps(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_repo_targets(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         # get the parents
         parent_target_repos = []
         if self.db_repo.parent:
             parents_query = Repository.query() \
                 .order_by(func.length(Repository.repo_name)) \
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
                 ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()
 
             for parent in parents:
                 parent_vcs_obj = parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     parent_target_repos.append(parent)
 
         # get other forks, and repo itself
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_id == self.db_repo.repo_id,  # repo itself
                     Repository.fork_id == self.db_repo.repo_id)  # forks of this repo
             )  \
             .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(Repository.repo_name.ilike(ilike_expression))
 
         limit = max(20 - len(parent_target_repos), 5)  # not less then 5
         target_repos = query.limit(limit).all()
 
         all_target_repos = target_repos + parent_target_repos
 
         repos = []
         # This checks permissions to the repositories
         for obj in ScmModel().get_repos(all_target_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @classmethod
     def get_comment_ids(cls, post_data):
         return filter(lambda e: e > 0, map(safe_int, aslist(post_data.get('comments'), ',')))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_comments(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         self.register_comments_vars(c, pull_request_latest, versions, include_drafts=False)
         all_comments = c.inline_comments_flat + c.comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_todos(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request, include_drafts=False)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request, include_drafts=False)
 
         all_comments = c.unresolved_comments + c.resolved_comments
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(c.unresolved_comments),
                        todo_comments=True, existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_drafts(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.draft_comments = CommentsModel() \
             .get_pull_request_drafts(self._rhodecode_db_user.user_id, pull_request)
 
         all_comments = c.draft_comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids, draft_comments=True)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
         common_ancestor_id = _form['common_ancestor']
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         if not (source_db_repo or target_db_repo):
             h.flash(_('source_repo or target repo not found'), category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_ref_obj = unicode_to_reference(source_ref)
         target_ref_obj = unicode_to_reference(target_ref)
 
         source_commit = source_scm.get_commit(source_ref_obj.commit_id)
         target_commit = target_scm.get_commit(target_ref_obj.commit_id)
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref = ':'.join((target_ref_obj.type, target_ref_obj.name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user,
             source_db_repo,
             source_ref_obj,
             target_db_repo,
             target_ref_obj,
             include_diff_info=False)
 
         reviewers = validate_default_reviewers(_form['review_members'], reviewer_rules)
         observers = validate_observers(_form['observer_members'], reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref_obj.name
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
         description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 common_ancestor_id=common_ancestor_id,
                 reviewers=reviewers,
                 observers=observers,
                 title=pullrequest_title,
                 description=description,
                 description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         c = self.load_default_context()
         redirect_url = None
         # we do this check as first, because we want to know ASAP in the flow that
         # pr is updating currently
         is_state_changing = pull_request.is_state_changing()
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return {'response': True,
                     'redirect_url': redirect_url}
 
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
 
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
             force_refresh = str2bool(self.request.POST.get('force_refresh', 'false'))
             do_update_commits = str2bool(self.request.POST.get('update_commits', 'false'))
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_REVIEWER)
             elif 'observer_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['observer_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_OBSERVER)
             elif do_update_commits:
                 if is_state_changing:
                     log.debug('commits update: forbidden because pull request is in state %s',
                               pull_request.pull_request_state)
                     msg = _(u'Cannot update pull requests commits in state other than `{}`. '
                             u'Current state is: `{}`').format(
                         PullRequest.STATE_CREATED, pull_request.pull_request_state)
                     h.flash(msg, category='error')
                     return {'response': True,
                             'redirect_url': redirect_url}
 
                 self._update_commits(c, pull_request)
                 if force_refresh:
                     redirect_url = h.route_path(
                         'pullrequest_show', repo_name=self.db_repo_name,
                         pull_request_id=pull_request.pull_request_id,
                         _query={"force_refresh": 1})
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 log.error('Unhandled update data.')
                 raise HTTPBadRequest()
 
             return {'response': True,
                     'redirect_url': redirect_url}
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         """
         Edit title and description
         """
         _ = self.request.translate
 
         try:
             PullRequestModel().edit(
                 pull_request,
                 self.request.POST.get('title'),
                 self.request.POST.get('description'),
                 self.request.POST.get('description_renderer'),
                 self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
     def _update_commits(self, c, pull_request):
         _ = self.request.translate
         log.debug('pull-request: running update commits actions')
 
         @retry(exception=Exception, n_tries=3, delay=2)
         def commits_update():
             return PullRequestModel().update_commits(
                 pull_request, self._rhodecode_db_user)
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             resp = commits_update()  # retry x3
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(u'Pull request updated to "{source_commit_id}" with '
                     u'{count_added} added, {count_removed} removed commits. '
                     u'Source of changes: {change_source}.')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     def _update_reviewers(self, c, pull_request, review_members, reviewer_rules, role):
         _ = self.request.translate
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         if role == PullRequestReviewers.ROLE_REVIEWER:
             try:
                 reviewers = validate_default_reviewers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Reviewers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             old_calculated_status = pull_request.calculated_review_status()
             PullRequestModel().update_reviewers(
                 pull_request, reviewers, self._rhodecode_db_user)
 
             Session().commit()
 
             msg = _('Pull request reviewers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
             # trigger status changed if change in reviewers changes the status
             calculated_status = pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
 
         elif role == PullRequestReviewers.ROLE_OBSERVER:
             try:
                 observers = validate_observers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Observers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             PullRequestModel().update_observers(
                 pull_request, observers, self._rhodecode_db_user)
 
             Session().commit()
             msg = _('Pull request observers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         if pull_request.is_state_changing():
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot merge pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_show',
                              repo_name=pull_request.target_repo.repo_name,
                              pull_request_id=pull_request.pull_request_id))
 
         self.load_default_context()
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             check = MergeCheck.validate(
                 pull_request, auth_user=self._rhodecode_user,
                 translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             with pull_request.set_state(PullRequest.STATE_UPDATING):
                 self._merge_pull_request(
                     pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s", merge_resp)
             msg = merge_resp.merge_status_message
             h.flash(msg, category='error')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     def _pull_request_comments_create(self, pull_request, comments):
         _ = self.request.translate
         data = {}
         if not comments:
             return
         pull_request_id = pull_request.pull_request_id
 
         all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
 
         for entry in comments:
             c = self.load_default_context()
             comment_type = entry['comment_type']
             text = entry['text']
             status = entry['status']
             is_draft = str2bool(entry['is_draft'])
             resolves_comment_id = entry['resolves_comment_id']
             close_pull_request = entry['close_pull_request']
             f_path = entry['f_path']
             line_no = entry['line']
             target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
 
             # the logic here should work like following, if we submit close
             # pr comment, use `close_pull_request_with_comment` function
             # else handle regular comment logic
 
             if close_pull_request:
                 # only owner or admin or person with write permissions
                 allowed_to_close = PullRequestModel().check_user_update(
                     pull_request, self._rhodecode_user)
                 if not allowed_to_close:
                     log.debug('comment: forbidden because not allowed to close '
                               'pull request %s', pull_request_id)
                     raise HTTPForbidden()
 
                 # This also triggers `review_status_change`
                 comment, status = PullRequestModel().close_pull_request_with_comment(
                     pull_request, self._rhodecode_user, self.db_repo, message=text,
                     auth_user=self._rhodecode_user)
                 Session().flush()
                 is_inline = comment.is_inline
 
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment',
                     data={'comment': comment})
 
             else:
                 # regular comment case, could be inline, or one with status.
                 # for that one we check also permissions
                 # Additionally ENSURE if somehow draft is sent we're then unable to change status
                 allowed_to_change_status = PullRequestModel().check_user_change_status(
                     pull_request, self._rhodecode_user) and not is_draft
 
                 if status and allowed_to_change_status:
                     message = (_('Status change %(transition_icon)s %(status)s')
                                % {'transition_icon': '>',
                                   'status': ChangesetStatus.get_status_lbl(status)})
                     text = text or message
 
                 comment = CommentsModel().create(
                     text=text,
                     repo=self.db_repo.repo_id,
                     user=self._rhodecode_user.user_id,
                     pull_request=pull_request,
                     f_path=f_path,
                     line_no=line_no,
                     status_change=(ChangesetStatus.get_status_lbl(status)
                                    if status and allowed_to_change_status else None),
                     status_change_type=(status
                                         if status and allowed_to_change_status else None),
                     comment_type=comment_type,
                     is_draft=is_draft,
                     resolves_comment_id=resolves_comment_id,
                     auth_user=self._rhodecode_user,
                     send_email=not is_draft,  # skip notification for draft comments
                 )
                 is_inline = comment.is_inline
 
                 if allowed_to_change_status:
                     # calculate old status before we change it
                     old_calculated_status = pull_request.calculated_review_status()
 
                     # get status if set !
                     if status:
                         ChangesetStatusModel().set_status(
                             self.db_repo.repo_id,
                             status,
                             self._rhodecode_user.user_id,
                             comment,
                             pull_request=pull_request
                         )
 
                     Session().flush()
                     # this is somehow required to get access to some relationship
                     # loaded on comment
                     Session().refresh(comment)
 
                     # skip notifications for drafts
                     if not is_draft:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'comment',
                             data={'comment': comment})
 
                     # we now calculate the status of pull request, and based on that
                     # calculation we set the commits status
                     calculated_status = pull_request.calculated_review_status()
                     if old_calculated_status != calculated_status:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'review_status_change',
                             data={'status': calculated_status})
 
             comment_id = comment.comment_id
             data[comment_id] = {
                 'target_id': target_elem_id
             }
             Session().flush()
 
             c.co = comment
             c.at_version_num = None
             c.is_new = True
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data[comment_id].update(comment.get_dict())
             data[comment_id].update({'rendered_text': rendered_comment})
 
         Session().commit()
 
         # skip channelstream for draft comments
         if not all_drafts:
             comment_broadcast_channel = channelstream.comment_channel(
                 self.db_repo_name, pull_request_obj=pull_request)
 
             comment_data = data
             posted_comment_type = 'inline' if is_inline else 'general'
             if len(data) == 1:
                 msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
             else:
                 msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
 
             channelstream.comment_channelstream_push(
                 self.request, comment_broadcast_channel, self._rhodecode_user, msg,
                 comment_data=comment_data)
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(self.request.matchdict['pull_request_id'])
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug('comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         comment_data = {
             'comment_type': self.request.POST.get('comment_type'),
             'text': self.request.POST.get('text'),
             'status': self.request.POST.get('changeset_status', None),
             'is_draft': self.request.POST.get('draft'),
             'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
             'close_pull_request': self.request.POST.get('close_pull_request'),
             'f_path': self.request.POST.get('f_path'),
             'line': self.request.POST.get('line'),
         }
         data = self._pull_request_comments_create(pull_request, [comment_data])
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if comment.draft and not comment_owner:
             # We never allow to delete draft comments for other than owners
             raise HTTPNotFound()
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_edit(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id']
         )
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id']
         )
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(PR): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(PR): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             Session().commit()
             if not comment.draft:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment_edit',
                     data={'comment': comment})
 
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()

rhodecode/apps/repository/views/repo_settings_issue_trackers.py

@@ -1,127 +1,125 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 import formencode
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.lib import audit_logger
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.model.forms import IssueTrackerPatternsForm
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSettingsIssueTrackersView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     def repo_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
         c.data = 'data'
 
         c.settings_model = self.db_repo_patterns
         c.global_patterns = c.settings_model.get_global_settings()
         c.repo_patterns = c.settings_model.get_repo_settings()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_test(self):
         return h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
             self.db_repo_name)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_delete(self):
         _ = self.request.translate
         uid = self.request.POST.get('uid')
         repo_settings = self.db_repo_patterns
         try:
             repo_settings.delete_entries(uid)
         except Exception:
             h.flash(_('Error occurred during deleting issue tracker entry'),
                     category='error')
             raise HTTPNotFound()
 
         SettingsModel().invalidate_settings_cache()
         h.flash(_('Removed issue tracker entry.'), category='success')
 
         return {'deleted': uid}
 
     def _update_patterns(self, form, repo_settings):
         for uid in form['delete_patterns']:
             repo_settings.delete_entries(uid)
 
         for pattern_data in form['patterns']:
             for setting_key, pattern, type_ in pattern_data:
                 sett = repo_settings.create_or_update_setting(
                     setting_key, pattern.strip(), type_)
                 Session().add(sett)
 
             Session().commit()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def repo_issuetracker_update(self):
         _ = self.request.translate
         # Save inheritance
         repo_settings = self.db_repo_patterns
         inherited = (
             self.request.POST.get('inherit_global_issuetracker') == "inherited")
         repo_settings.inherit_global_settings = inherited
         Session().commit()
 
         try:
             form = IssueTrackerPatternsForm(self.request.translate)().to_python(self.request.POST)
         except formencode.Invalid as errors:
             log.exception('Failed to add new pattern')
             error = errors
             h.flash(_('Invalid issue tracker pattern: {}'.format(error)),
                     category='error')
             raise HTTPFound(
                 h.route_path('edit_repo_issuetracker',
                              repo_name=self.db_repo_name))
 
         if form:
             self._update_patterns(form, repo_settings)
 
         h.flash(_('Updated issue tracker entries'), category='success')
         raise HTTPFound(
             h.route_path('edit_repo_issuetracker', repo_name=self.db_repo_name))
 

rhodecode/apps/repository/views/repo_summary.py

@@ -1,290 +1,290 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import string
 import time
 
 import rhodecode
 
 
 
 from rhodecode.lib.view_utils import get_format_ref_id
 from rhodecode.apps._base import RepoAppView
 from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
 from rhodecode.lib import helpers as h, rc_cache
 from rhodecode.lib.utils2 import safe_str, safe_int
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     CommitError, EmptyRepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import Statistics, CacheKey, User
 from rhodecode.model.meta import Session
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSummaryView(RepoAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = None
         if not c.repository_requirements_missing:
             c.rhodecode_repo = self.rhodecode_vcs_repo
         return c
 
     def _load_commits_context(self, c):
         p = safe_int(self.request.GET.get('page'), 1)
         size = safe_int(self.request.GET.get('size'), 10)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num,
                 'size': size
             }
             return h.route_path(
                 'repo_summary_commits',
                 repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)
 
         pre_load = self.get_commit_preload_attrs()
 
         try:
             collection = self.rhodecode_vcs_repo.get_commits(
                 pre_load=pre_load, translate_tags=False)
         except EmptyRepositoryError:
             collection = self.rhodecode_vcs_repo
 
         c.repo_commits = h.RepoPage(
             collection, page=p, items_per_page=size, url_maker=url_generator)
         page_ids = [x.raw_id for x in c.repo_commits]
         c.comments = self.db_repo.get_comments(page_ids)
         c.statuses = self.db_repo.statuses(page_ids)
 
     def _prepare_and_set_clone_url(self, c):
         username = ''
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = safe_str(self._rhodecode_user.username)
 
         _def_clone_uri = c.clone_uri_tmpl
         _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri_id)
         c.clone_repo_url_ssh = self.db_repo.clone_url(
             uri_tmpl=_def_clone_uri_ssh, ssh=True)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def summary_commits(self):
         c = self.load_default_context()
         self._prepare_and_set_clone_url(c)
         self._load_commits_context(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def summary(self):
         c = self.load_default_context()
 
         # Prepare the clone URL
         self._prepare_and_set_clone_url(c)
 
         # If enabled, get statistics data
         c.show_stats = bool(self.db_repo.enable_statistics)
 
         stats = Session().query(Statistics) \
             .filter(Statistics.repository == self.db_repo) \
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is self.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
 
             # Sort first by decreasing count and second by the file extension,
             # so we have a consistent output.
             lang_stats_items = sorted(lang_stats_d.items(),
                                       key=lambda k: (-k[1], k[0]))[:10]
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_items]
 
             c.trending_languages = json.dumps(lang_stats)
         else:
             c.no_data = True
             c.trending_languages = json.dumps({})
 
         scm_model = ScmModel()
         c.enable_downloads = self.db_repo.enable_downloads
         c.repository_followers = scm_model.get_followers(self.db_repo)
         c.repository_forks = scm_model.get_forks(self.db_repo)
 
         # first interaction with the VCS instance after here...
         if c.repository_requirements_missing:
             self.request.override_renderer = \
                 'rhodecode:templates/summary/missing_requirements.mako'
             return self._get_template_context(c)
 
         c.readme_data, c.readme_file = \
             self._get_readme_data(self.db_repo, c.visual.default_renderer)
 
         # loads the summary commits template context
         self._load_commits_context(c)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_stats(self):
         show_stats = bool(self.db_repo.enable_statistics)
         repo_id = self.db_repo.repo_id
 
         landing_commit = self.db_repo.get_landing_commit()
         if isinstance(landing_commit, EmptyCommit):
             return {'size': 0, 'code_stats': {}}
 
         cache_seconds = safe_int(rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = cache_seconds > 0
 
         log.debug(
             'Computing REPO STATS for repo_id %s commit_id `%s` '
             'with caching: %s[TTL: %ss]' % (
                 repo_id, landing_commit, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
         def compute_stats(repo_id, commit_id, _show_stats):
             code_stats = {}
             size = 0
             try:
                 commit = self.db_repo.get_commit(commit_id)
 
                 for node in commit.get_filenodes_generator():
                     size += node.size
                     if not _show_stats:
                         continue
-                    ext = string.lower(node.extension)
+                    ext = node.extension.lower()
                     ext_info = LANGUAGES_EXTENSIONS_MAP.get(ext)
                     if ext_info:
                         if ext in code_stats:
                             code_stats[ext]['count'] += 1
                         else:
                             code_stats[ext] = {"count": 1, "desc": ext_info}
             except (EmptyRepositoryError, CommitDoesNotExistError):
                 pass
             return {'size': h.format_byte_size_binary(size),
                     'code_stats': code_stats}
 
         stats = compute_stats(self.db_repo.repo_id, landing_commit.raw_id, show_stats)
         return stats
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_refs_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
         refs_to_create = [
             (_("Branch"), repo.branches, 'branch'),
             (_("Tag"), repo.tags, 'tag'),
             (_("Bookmark"), repo.bookmarks, 'book'),
         ]
         res = self._create_reference_data(repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_refs_changelog_data(self):
         _ = self.request.translate
         self.load_default_context()
 
         repo = self.rhodecode_vcs_repo
 
         refs_to_create = [
             (_("Branches"), repo.branches, 'branch'),
             (_("Closed branches"), repo.branches_closed, 'branch_closed'),
             # TODO: enable when vcs can handle bookmarks filters
             # (_("Bookmarks"), repo.bookmarks, "book"),
         ]
         res = self._create_reference_data(
             repo, self.db_repo_name, refs_to_create)
         data = {
             'more': False,
             'results': res
         }
         return data
 
     def _create_reference_data(self, repo, full_repo_name, refs_to_create):
         format_ref_id = get_format_ref_id(repo)
 
         result = []
         for title, refs, ref_type in refs_to_create:
             if refs:
                 result.append({
                     'text': title,
                     'children': self._create_reference_items(
                         repo, full_repo_name, refs, ref_type,
                         format_ref_id),
                 })
         return result
 
     def _create_reference_items(self, repo, full_repo_name, refs, ref_type, format_ref_id):
         result = []
         is_svn = h.is_svn(repo)
         for ref_name, raw_id in refs.items():
             files_url = self._create_files_url(
                 repo, full_repo_name, ref_name, raw_id, is_svn)
             result.append({
                 'text': ref_name,
                 'id': format_ref_id(ref_name, raw_id),
                 'raw_id': raw_id,
                 'type': ref_type,
                 'files_url': files_url,
                 'idx': 0,
             })
         return result
 
     def _create_files_url(self, repo, full_repo_name, ref_name, raw_id, is_svn):
         use_commit_id = '/' in ref_name or is_svn
         return h.route_path(
             'repo_files',
             repo_name=full_repo_name,
             f_path=ref_name if is_svn else '',
             commit_id=raw_id if use_commit_id else ref_name,
             _query=dict(at=ref_name))

rhodecode/authentication/plugins/auth_pam.py

@@ -1,172 +1,172 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication library for PAM
 """
 
 import colander
 import grp
 import logging
 import pam
 import pwd
 import re
 import socket
 
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, hybrid_property)
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwargs):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class PamAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class PamSettingsSchema(AuthnPluginSettingsSchemaBase):
     service = colander.SchemaNode(
         colander.String(),
         default='login',
         description=_('PAM service name to use for authentication.'),
         preparer=strip_whitespace,
         title=_('PAM service name'),
         widget='string')
     gecos = colander.SchemaNode(
         colander.String(),
-        default='(?P<last_name>.+),\s*(?P<first_name>\w+)',
+        default=r'(?P<last_name>.+),\s*(?P<first_name>\w+)',
         description=_('Regular expression for extracting user name/email etc. '
                       'from Unix userinfo.'),
         preparer=strip_whitespace,
         title=_('Gecos Regex'),
         widget='string')
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
     uid = 'pam'
     # PAM authentication can be slow. Repository operations involve a lot of
     # auth calls. Little caching helps speedup push/pull operations significantly
     AUTH_CACHE_TTL = 4
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), PamAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
             renderer='rhodecode:templates/admin/auth/plugin_settings.mako',
             request_method='GET',
             route_name='auth_home',
             context=PamAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
             renderer='rhodecode:templates/admin/auth/plugin_settings.mako',
             request_method='POST',
             route_name='auth_home',
             context=PamAuthnResource)
 
     def get_display_name(self, load_from_settings=False):
         return _('PAM')
 
     @classmethod
     def docs(cls):
         return "https://docs.rhodecode.com/RhodeCode-Enterprise/auth/auth-pam.html"
 
     @hybrid_property
     def name(self):
         return u"pam"
 
     def get_settings_schema(self):
         return PamSettingsSchema()
 
     def use_fake_password(self):
         return True
 
     def auth(self, userobj, username, password, settings, **kwargs):
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
         _pam = pam.pam()
         auth_result = _pam.authenticate(username, password, settings["service"])
 
         if not auth_result:
             log.error("PAM was unable to authenticate user: %s", username)
             return None
 
         log.debug('Got PAM response %s', auth_result)
 
         # old attrs fetched from RhodeCode database
         default_email = "%s@%s" % (username, socket.gethostname())
         admin = getattr(userobj, 'admin', False)
         active = getattr(userobj, 'active', True)
         email = getattr(userobj, 'email', '') or default_email
         username = getattr(userobj, 'username', username)
         firstname = getattr(userobj, 'firstname', '')
         lastname = getattr(userobj, 'lastname', '')
         extern_type = getattr(userobj, 'extern_type', '')
 
         user_attrs = {
             'username': username,
             'firstname': firstname,
             'lastname': lastname,
             'groups': [g.gr_name for g in grp.getgrall()
                        if username in g.gr_mem],
             'user_group_sync': True,
             'email': email,
             'admin': admin,
             'active': active,
             'active_from_extern': None,
             'extern_name': username,
             'extern_type': extern_type,
         }
 
         try:
             user_data = pwd.getpwnam(username)
             regex = settings["gecos"]
             match = re.search(regex, user_data.pw_gecos)
             if match:
                 user_attrs["firstname"] = match.group('first_name')
                 user_attrs["lastname"] = match.group('last_name')
         except Exception:
             log.warning("Cannot extract additional info for PAM user")
             pass
 
         log.debug("pamuser: %s", user_attrs)
         log.info('user `%s` authenticated correctly', user_attrs['username'],
                  extra={"action": "user_auth_ok", "auth_module": "auth_pam", "username": user_attrs["username"]})
         return user_attrs
 
 
 def includeme(config):
     plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
     plugin_factory(plugin_id).includeme(config)

rhodecode/config/environment.py

@@ -1,89 +1,90 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import logging
 import rhodecode
+import collections
 
 from rhodecode.config import utils
 
 from rhodecode.lib.utils import load_rcextensions
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib.vcs import connect_vcs
 
 log = logging.getLogger(__name__)
 
 
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
 
     # TODO(marcink): probably not required anymore
     # configure channelstream,
     settings_merged['channelstream_config'] = {
         'enabled': str2bool(settings_merged.get('channelstream.enabled', False)),
         'server': settings_merged.get('channelstream.server'),
         'secret': settings_merged.get('channelstream.secret')
     }
 
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
     if settings['is_test']:
         rhodecode.is_test = True
         rhodecode.disable_error_handler = True
         from rhodecode import authentication
         authentication.plugin_default_auth_ttl = 0
 
         utils.initialize_test_environment(settings_merged)
 
     # Initialize the database connection.
     utils.initialize_database(settings_merged)
 
     load_rcextensions(root_path=settings_merged['here'])
 
     # Limit backends to `vcs.backends` from configuration, and preserve the order
     for alias in rhodecode.BACKENDS.keys():
         if alias not in settings['vcs.backends']:
             del rhodecode.BACKENDS[alias]
 
     _sorted_backend = sorted(rhodecode.BACKENDS.items(),
                              key=lambda item: settings['vcs.backends'].index(item[0]))
-    rhodecode.BACKENDS = rhodecode.OrderedDict(_sorted_backend)
+    rhodecode.BACKENDS = collections.OrderedDict(_sorted_backend)
 
     log.info('Enabled VCS backends: %s', rhodecode.BACKENDS.keys())
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = settings['vcs.server']
     vcs_server_enabled = settings['vcs.server.enable']
 
     utils.configure_vcs(settings)
 
     # Store the settings to make them available to other modules.
 
     rhodecode.PYRAMID_SETTINGS = settings_merged
     rhodecode.CONFIG = settings_merged
     rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
     else:
         log.warning('vcs-server not enabled, vcs connection unavailable')

rhodecode/lib/_vendor/authomatic/adapters.py

@@ -1,282 +1,280 b''
 # -*- coding: utf-8 -*-
 """
 Adapters
 --------
 
 .. contents::
    :backlinks: none
 
 The :func:`authomatic.login` function needs access to functionality like
 getting the **URL** of the handler where it is being called, getting the
 **request params** and **cookies** and **writing the body**, **headers**
 and **status** to the response.
 
 Since implementation of these features varies across Python web frameworks,
 the Authomatic library uses **adapters** to unify these differences into a
 single interface.
 
 Available Adapters
 ^^^^^^^^^^^^^^^^^^
 
 If you are missing an adapter for the framework of your choice, please
 open an `enhancement issue <https://github.com/authomatic/authomatic/issues>`_
 or consider a contribution to this module by
 :ref:`implementing <implement_adapters>` one by yourself.
 Its very easy and shouldn't take you more than a few minutes.
 
 .. autoclass:: DjangoAdapter
     :members:
 
 .. autoclass:: Webapp2Adapter
     :members:
 
 .. autoclass:: WebObAdapter
     :members:
 
 .. autoclass:: WerkzeugAdapter
     :members:
 
 .. _implement_adapters:
 
 Implementing an Adapter
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 Implementing an adapter for a Python web framework is pretty easy.
 
 Do it by subclassing the :class:`.BaseAdapter` abstract class.
 There are only **six** members that you need to implement.
 
 Moreover if your framework is based on the |webob|_ or |werkzeug|_ package
 you can subclass the :class:`.WebObAdapter` or :class:`.WerkzeugAdapter`
 respectively.
 
 .. autoclass:: BaseAdapter
     :members:
 
 """
 
 import abc
 from authomatic.core import Response
 
 
-class BaseAdapter(object):
+class BaseAdapter(object, metaclass=abc.ABCMeta):
     """
     Base class for platform adapters.
 
     Defines common interface for WSGI framework specific functionality.
 
     """
 
-    __metaclass__ = abc.ABCMeta
-
     @abc.abstractproperty
     def params(self):
         """
         Must return a :class:`dict` of all request parameters of any HTTP
         method.
 
         :returns:
             :class:`dict`
 
         """
 
     @abc.abstractproperty
     def url(self):
         """
         Must return the url of the actual request including path but without
         query and fragment.
 
         :returns:
             :class:`str`
 
         """
 
     @abc.abstractproperty
     def cookies(self):
         """
         Must return cookies as a :class:`dict`.
 
         :returns:
             :class:`dict`
 
         """
 
     @abc.abstractmethod
     def write(self, value):
         """
         Must write specified value to response.
 
         :param str value:
             String to be written to response.
 
         """
 
     @abc.abstractmethod
     def set_header(self, key, value):
         """
         Must set response headers to ``Key: value``.
 
         :param str key:
             Header name.
 
         :param str value:
             Header value.
 
         """
 
     @abc.abstractmethod
     def set_status(self, status):
         """
         Must set the response status e.g. ``'302 Found'``.
 
         :param str status:
             The HTTP response status.
 
         """
 
 
 class DjangoAdapter(BaseAdapter):
     """
     Adapter for the |django|_ framework.
     """
 
     def __init__(self, request, response):
         """
         :param request:
             An instance of the :class:`django.http.HttpRequest` class.
 
         :param response:
             An instance of the :class:`django.http.HttpResponse` class.
         """
         self.request = request
         self.response = response
 
     @property
     def params(self):
         params = {}
         params.update(self.request.GET.dict())
         params.update(self.request.POST.dict())
         return params
 
     @property
     def url(self):
         return self.request.build_absolute_uri(self.request.path)
 
     @property
     def cookies(self):
         return dict(self.request.COOKIES)
 
     def write(self, value):
         self.response.write(value)
 
     def set_header(self, key, value):
         self.response[key] = value
 
     def set_status(self, status):
         status_code, reason = status.split(' ', 1)
         self.response.status_code = int(status_code)
 
 
 class WebObAdapter(BaseAdapter):
     """
     Adapter for the |webob|_ package.
     """
 
     def __init__(self, request, response):
         """
         :param request:
             A |webob|_ :class:`Request` instance.
 
         :param response:
             A |webob|_ :class:`Response` instance.
         """
         self.request = request
         self.response = response
 
     # =========================================================================
     # Request
     # =========================================================================
 
     @property
     def url(self):
         return self.request.path_url
 
     @property
     def params(self):
         return dict(self.request.params)
 
     @property
     def cookies(self):
         return dict(self.request.cookies)
 
     # =========================================================================
     # Response
     # =========================================================================
 
     def write(self, value):
         self.response.write(value)
 
     def set_header(self, key, value):
         self.response.headers[key] = str(value)
 
     def set_status(self, status):
         self.response.status = status
 
 
 class Webapp2Adapter(WebObAdapter):
     """
     Adapter for the |webapp2|_ framework.
 
     Inherits from the :class:`.WebObAdapter`.
 
     """
 
     def __init__(self, handler):
         """
         :param handler:
             A :class:`webapp2.RequestHandler` instance.
         """
         self.request = handler.request
         self.response = handler.response
 
 
 class WerkzeugAdapter(BaseAdapter):
     """
     Adapter for |flask|_ and other |werkzeug|_ based frameworks.
 
     Thanks to `Mark Steve Samson <http://marksteve.com>`_.
 
     """
 
     @property
     def params(self):
         return self.request.args
 
     @property
     def url(self):
         return self.request.base_url
 
     @property
     def cookies(self):
         return self.request.cookies
 
     def __init__(self, request, response):
         """
         :param request:
             Instance of the :class:`werkzeug.wrappers.Request` class.
 
         :param response:
             Instance of the :class:`werkzeug.wrappers.Response` class.
         """
 
         self.request = request
         self.response = response
 
     def write(self, value):
         self.response.data = self.response.data + value
 
     def set_header(self, key, value):
         self.response.headers[key] = value
 
     def set_status(self, status):
         self.response.status = status

rhodecode/lib/audit_logger.py

@@ -1,305 +1,305 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 
 from rhodecode.lib.jsonalchemy import JsonRaw
 from rhodecode.model import meta
 from rhodecode.model.db import User, UserLog, Repository
 
 
 log = logging.getLogger(__name__)
 
 # action as key, and expected action_data as value
 ACTIONS_V1 = {
     'user.login.success': {'user_agent': ''},
     'user.login.failure': {'user_agent': ''},
     'user.logout': {'user_agent': ''},
     'user.register': {},
     'user.password.reset_request': {},
     'user.push': {'user_agent': '', 'commit_ids': []},
     'user.pull': {'user_agent': ''},
 
     'user.create': {'data': {}},
     'user.delete': {'old_data': {}},
     'user.edit': {'old_data': {}},
     'user.edit.permissions': {},
     'user.edit.ip.add': {'ip': {}, 'user': {}},
     'user.edit.ip.delete': {'ip': {}, 'user': {}},
     'user.edit.token.add': {'token': {}, 'user': {}},
     'user.edit.token.delete': {'token': {}, 'user': {}},
     'user.edit.email.add': {'email': ''},
     'user.edit.email.delete': {'email': ''},
     'user.edit.ssh_key.add': {'token': {}, 'user': {}},
     'user.edit.ssh_key.delete': {'token': {}, 'user': {}},
     'user.edit.password_reset.enabled': {},
     'user.edit.password_reset.disabled': {},
 
     'user_group.create': {'data': {}},
     'user_group.delete': {'old_data': {}},
     'user_group.edit': {'old_data': {}},
     'user_group.edit.permissions': {},
     'user_group.edit.member.add': {'user': {}},
     'user_group.edit.member.delete': {'user': {}},
 
     'repo.create': {'data': {}},
     'repo.fork': {'data': {}},
     'repo.edit': {'old_data': {}},
     'repo.edit.permissions': {},
     'repo.edit.permissions.branch': {},
     'repo.archive': {'old_data': {}},
     'repo.delete': {'old_data': {}},
 
     'repo.archive.download': {'user_agent': '', 'archive_name': '',
                               'archive_spec': '', 'archive_cached': ''},
 
     'repo.permissions.branch_rule.create': {},
     'repo.permissions.branch_rule.edit': {},
     'repo.permissions.branch_rule.delete': {},
 
     'repo.pull_request.create': '',
     'repo.pull_request.edit': '',
     'repo.pull_request.delete': '',
     'repo.pull_request.close': '',
     'repo.pull_request.merge': '',
     'repo.pull_request.vote': '',
     'repo.pull_request.comment.create': '',
     'repo.pull_request.comment.edit': '',
     'repo.pull_request.comment.delete': '',
 
     'repo.pull_request.reviewer.add': '',
     'repo.pull_request.reviewer.delete': '',
 
     'repo.pull_request.observer.add': '',
     'repo.pull_request.observer.delete': '',
 
     'repo.commit.strip': {'commit_id': ''},
     'repo.commit.comment.create': {'data': {}},
     'repo.commit.comment.delete': {'data': {}},
     'repo.commit.comment.edit': {'data': {}},
     'repo.commit.vote': '',
 
     'repo.artifact.add': '',
     'repo.artifact.delete': '',
 
     'repo_group.create': {'data': {}},
     'repo_group.edit': {'old_data': {}},
     'repo_group.edit.permissions': {},
     'repo_group.delete': {'old_data': {}},
 }
 
 ACTIONS = ACTIONS_V1
 
 SOURCE_WEB = 'source_web'
 SOURCE_API = 'source_api'
 
 
 class UserWrap(object):
     """
     Fake object used to imitate AuthUser
     """
 
     def __init__(self, user_id=None, username=None, ip_addr=None):
         self.user_id = user_id
         self.username = username
         self.ip_addr = ip_addr
 
 
 class RepoWrap(object):
     """
     Fake object used to imitate RepoObject that audit logger requires
     """
 
     def __init__(self, repo_id=None, repo_name=None):
         self.repo_id = repo_id
         self.repo_name = repo_name
 
 
 def _store_log(action_name, action_data, user_id, username, user_data,
                ip_address, repository_id, repository_name):
     user_log = UserLog()
     user_log.version = UserLog.VERSION_2
 
     user_log.action = action_name
-    user_log.action_data = action_data or JsonRaw(u'{}')
+    user_log.action_data = action_data or JsonRaw('{}')
 
     user_log.user_ip = ip_address
 
     user_log.user_id = user_id
     user_log.username = username
-    user_log.user_data = user_data or JsonRaw(u'{}')
+    user_log.user_data = user_data or JsonRaw('{}')
 
     user_log.repository_id = repository_id
     user_log.repository_name = repository_name
 
     user_log.action_date = datetime.datetime.now()
 
     return user_log
 
 
 def store_web(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_WEB
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store_api(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_API
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store(action, user, action_data=None, user_data=None, ip_addr=None,
           repo=None, sa_session=None, commit=False):
     """
     Audit logger for various actions made by users, typically this
     results in a call such::
 
         from rhodecode.lib import audit_logger
 
         audit_logger.store(
             'repo.edit', user=self._rhodecode_user)
         audit_logger.store(
             'repo.delete', action_data={'data': repo_data},
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'))
 
         # repo action
         audit_logger.store(
             'repo.delete',
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'),
             repo=audit_logger.RepoWrap(repo_name='some-repo'))
 
         # repo action, when we know and have the repository object already
         audit_logger.store(
             'repo.delete', action_data={'source': audit_logger.SOURCE_WEB, },
             user=self._rhodecode_user,
             repo=repo_object)
 
         # alternative wrapper to the above
         audit_logger.store_web(
             'repo.delete', action_data={},
             user=self._rhodecode_user,
             repo=repo_object)
 
         # without an user ?
         audit_logger.store(
             'user.login.failure',
             user=audit_logger.UserWrap(
                     username=self.request.params.get('username'),
                     ip_addr=self.request.remote_addr))
 
     """
     from rhodecode.lib.utils2 import safe_unicode
     from rhodecode.lib.auth import AuthUser
 
     action_spec = ACTIONS.get(action, None)
     if action_spec is None:
         raise ValueError('Action `{}` is not supported'.format(action))
 
     if not sa_session:
         sa_session = meta.Session()
 
     try:
         username = getattr(user, 'username', None)
         if not username:
             pass
 
         user_id = getattr(user, 'user_id', None)
         if not user_id:
             # maybe we have username ? Try to figure user_id from username
             if username:
                 user_id = getattr(
                     User.get_by_username(username), 'user_id', None)
 
         ip_addr = ip_addr or getattr(user, 'ip_addr', None)
         if not ip_addr:
             pass
 
         if not user_data:
             # try to get this from the auth user
             if isinstance(user, AuthUser):
                 user_data = {
                     'username': user.username,
                     'email': user.email,
                 }
 
         repository_name = getattr(repo, 'repo_name', None)
         repository_id = getattr(repo, 'repo_id', None)
         if not repository_id:
             # maybe we have repo_name ? Try to figure repo_id from repo_name
             if repository_name:
                 repository_id = getattr(
                     Repository.get_by_repo_name(repository_name), 'repo_id', None)
 
         action_name = safe_unicode(action)
         ip_address = safe_unicode(ip_addr)
 
         with sa_session.no_autoflush:
 
             user_log = _store_log(
                 action_name=action_name,
                 action_data=action_data or {},
                 user_id=user_id,
                 username=username,
                 user_data=user_data or {},
                 ip_address=ip_address,
                 repository_id=repository_id,
                 repository_name=repository_name
             )
 
             sa_session.add(user_log)
             if commit:
                 sa_session.commit()
             entry_id = user_log.entry_id or ''
 
             update_user_last_activity(sa_session, user_id)
 
             if commit:
                 sa_session.commit()
 
         log.info('AUDIT[%s]: Logging action: `%s` by user:id:%s[%s] ip:%s',
                  entry_id, action_name, user_id, username, ip_address,
                  extra={"entry_id": entry_id, "action": action_name,
                         "user_id": user_id, "ip": ip_address})
 
     except Exception:
         log.exception('AUDIT: failed to store audit log')
 
 
 def update_user_last_activity(sa_session, user_id):
     _last_activity = datetime.datetime.now()
     try:
         sa_session.query(User).filter(User.user_id == user_id).update(
             {"last_activity": _last_activity})
         log.debug(
             'updated user `%s` last activity to:%s', user_id, _last_activity)
     except Exception:
         log.exception("Failed last activity update for user_id: %s", user_id)
         sa_session.rollback()
 

rhodecode/lib/channelstream.py

@@ -1,371 +1,371 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import itsdangerous
 import logging
 import requests
 import datetime
 
 from dogpile.util.readwrite_lock import ReadWriteMutex
 from pyramid.threadlocal import get_current_registry
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.ext_json import json
 from rhodecode.model.db import User
 
 log = logging.getLogger(__name__)
 
 LOCK = ReadWriteMutex()
 
 USER_STATE_PUBLIC_KEYS = [
     'id', 'username', 'first_name', 'last_name',
     'icon_link', 'display_name', 'display_link']
 
 
 class ChannelstreamException(Exception):
     pass
 
 
 class ChannelstreamConnectionException(ChannelstreamException):
     pass
 
 
 class ChannelstreamPermissionException(ChannelstreamException):
     pass
 
 
 def get_channelstream_server_url(config, endpoint):
     return 'http://{}{}'.format(config['server'], endpoint)
 
 
 def channelstream_request(config, payload, endpoint, raise_exc=True):
     signer = itsdangerous.TimestampSigner(config['secret'])
     sig_for_server = signer.sign(endpoint)
     secret_headers = {'x-channelstream-secret': sig_for_server,
                       'x-channelstream-endpoint': endpoint,
                       'Content-Type': 'application/json'}
     req_url = get_channelstream_server_url(config, endpoint)
 
     log.debug('Sending a channelstream request to endpoint: `%s`', req_url)
     response = None
     try:
         response = requests.post(req_url, data=json.dumps(payload),
                                  headers=secret_headers).json()
     except requests.ConnectionError:
         log.exception('ConnectionError occurred for endpoint %s', req_url)
         if raise_exc:
             raise ChannelstreamConnectionException(req_url)
     except Exception:
         log.exception('Exception related to Channelstream happened')
         if raise_exc:
             raise ChannelstreamConnectionException()
     log.debug('Got channelstream response: %s', response)
     return response
 
 
 def get_user_data(user_id):
     user = User.get(user_id)
     return {
         'id': user.user_id,
         'username': user.username,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'icon_link': h.gravatar_url(user.email, 60),
         'display_name': h.person(user, 'username_or_name_or_email'),
         'display_link': h.link_to_user(user),
         'notifications': user.user_data.get('notification_status', True)
     }
 
 
 def broadcast_validator(channel_name):
     """ checks if user can access the broadcast channel """
     if channel_name == 'broadcast':
         return True
 
 
 def repo_validator(channel_name):
     """ checks if user can access the broadcast channel """
     channel_prefix = '/repo$'
     if channel_name.startswith(channel_prefix):
         elements = channel_name[len(channel_prefix):].split('$')
         repo_name = elements[0]
         can_access = HasRepoPermissionAny(
             'repository.read',
             'repository.write',
             'repository.admin')(repo_name)
         log.debug(
             'permission check for %s channel resulted in %s',
             repo_name, can_access)
         if can_access:
             return True
     return False
 
 
 def check_channel_permissions(channels, plugin_validators, should_raise=True):
     valid_channels = []
 
     validators = [broadcast_validator, repo_validator]
     if plugin_validators:
         validators.extend(plugin_validators)
     for channel_name in channels:
         is_valid = False
         for validator in validators:
             if validator(channel_name):
                 is_valid = True
                 break
         if is_valid:
             valid_channels.append(channel_name)
         else:
             if should_raise:
                 raise ChannelstreamPermissionException()
     return valid_channels
 
 
 def get_channels_info(self, channels):
     payload = {'channels': channels}
     # gather persistence info
     return channelstream_request(self._config(), payload, '/info')
 
 
 def parse_channels_info(info_result, include_channel_info=None):
     """
     Returns data that contains only secure information that can be
     presented to clients
     """
     include_channel_info = include_channel_info or []
 
     user_state_dict = {}
     for userinfo in info_result['users']:
         user_state_dict[userinfo['user']] = {
             k: v for k, v in userinfo['state'].items()
             if k in USER_STATE_PUBLIC_KEYS
             }
 
     channels_info = {}
 
     for c_name, c_info in info_result['channels'].items():
         if c_name not in include_channel_info:
             continue
         connected_list = []
         for username in c_info['users']:
             connected_list.append({
                 'user': username,
                 'state': user_state_dict[username]
             })
         channels_info[c_name] = {'users': connected_list,
                                  'history': c_info['history']}
 
     return channels_info
 
 
 def log_filepath(history_location, channel_name):
     hasher = hashlib.sha256()
     hasher.update(channel_name.encode('utf8'))
     filename = '{}.log'.format(hasher.hexdigest())
     filepath = os.path.join(history_location, filename)
     return filepath
 
 
 def read_history(history_location, channel_name):
     filepath = log_filepath(history_location, channel_name)
     if not os.path.exists(filepath):
         return []
     history_lines_limit = -100
     history = []
     with open(filepath, 'rb') as f:
         for line in f.readlines()[history_lines_limit:]:
             try:
                 history.append(json.loads(line))
             except Exception:
                 log.exception('Failed to load history')
     return history
 
 
 def update_history_from_logs(config, channels, payload):
     history_location = config.get('history.location')
     for channel in channels:
         history = read_history(history_location, channel)
         payload['channels_info'][channel]['history'] = history
 
 
 def write_history(config, message):
     """ writes a message to a base64encoded filename """
     history_location = config.get('history.location')
     if not os.path.exists(history_location):
         return
     try:
         LOCK.acquire_write_lock()
         filepath = log_filepath(history_location, message['channel'])
         json_message = json.dumps(message)
         with open(filepath, 'ab') as f:
             f.write(json_message)
             f.write('\n')
     finally:
         LOCK.release_write_lock()
 
 
 def get_connection_validators(registry):
     validators = []
     for k, config in registry.rhodecode_plugins.items():
         validator = config.get('channelstream', {}).get('connect_validator')
         if validator:
             validators.append(validator)
     return validators
 
 
 def get_channelstream_config(registry=None):
     if not registry:
         registry = get_current_registry()
 
     rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
     channelstream_config = rhodecode_plugins.get('channelstream', {})
     return channelstream_config
 
 
 def post_message(channel, message, username, registry=None):
     channelstream_config = get_channelstream_config(registry)
     if not channelstream_config.get('enabled'):
         return
 
     message_obj = message
     if isinstance(message, str):
         message_obj = {
             'message': message,
             'level': 'success',
             'topic': '/notifications'
         }
 
     log.debug('Channelstream: sending notification to channel %s', channel)
     payload = {
         'type': 'message',
         'timestamp': datetime.datetime.utcnow(),
         'user': 'system',
         'exclude_users': [username],
         'channel': channel,
         'message': message_obj
     }
 
     try:
         return channelstream_request(
             channelstream_config, [payload], '/message',
             raise_exc=False)
     except ChannelstreamException:
         log.exception('Failed to send channelstream data')
         raise
 
 
 def _reload_link(label):
     return (
         '<a onclick="window.location.reload()">'
         '<strong>{}</strong>'
         '</a>'.format(label)
     )
 
 
 def pr_channel(pull_request):
     repo_name = pull_request.target_repo.repo_name
     pull_request_id = pull_request.pull_request_id
     channel = '/repo${}$/pr/{}'.format(repo_name, pull_request_id)
     log.debug('Getting pull-request channelstream broadcast channel: %s', channel)
     return channel
 
 
 def comment_channel(repo_name, commit_obj=None, pull_request_obj=None):
     channel = None
     if commit_obj:
-        channel = u'/repo${}$/commit/{}'.format(
+        channel = '/repo${}$/commit/{}'.format(
             repo_name, commit_obj.raw_id
         )
     elif pull_request_obj:
-        channel = u'/repo${}$/pr/{}'.format(
+        channel = '/repo${}$/pr/{}'.format(
             repo_name, pull_request_obj.pull_request_id
         )
     log.debug('Getting comment channelstream broadcast channel: %s', channel)
 
     return channel
 
 
 def pr_update_channelstream_push(request, pr_broadcast_channel, user, msg, **kwargs):
     """
     Channel push on pull request update
     """
     if not pr_broadcast_channel:
         return
 
     _ = request.translate
 
     message = '{} {}'.format(
         msg,
         _reload_link(_(' Reload page to load changes')))
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         pr_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
 
 def comment_channelstream_push(request, comment_broadcast_channel, user, msg, **kwargs):
     """
     Channelstream push on comment action, on commit, or pull-request
     """
     if not comment_broadcast_channel:
         return
 
     _ = request.translate
 
     comment_data = kwargs.pop('comment_data', {})
     user_data = kwargs.pop('user_data', {})
     comment_id = comment_data.keys()[0] if comment_data else ''
 
     message = '<strong>{}</strong> {} #{}'.format(
         user.username,
         msg,
         comment_id,
     )
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
     message_obj = {
         'message': None,
         'user': user.username,
         'comment_id': comment_id,
         'comment_data': comment_data,
         'user_data': user_data,
         'topic': '/comment'
     }
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)

rhodecode/lib/codeblocks.py

@@ -1,797 +1,797 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
 from itertools import groupby
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from pygments.lexers.special import TextLexer, Token
 from pygments.lexers import get_lexer_by_name
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict, safe_unicode
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer, DEL_FILENODE, BIN_FILENODE
 
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
     log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
               lexer, filenode, org_lexer)
     content = filenode.content
     tokens = tokenize_string(content, lexer)
     lines = split_token_stream(tokens, content)
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
 
     if isinstance(lexer, TextLexer):
         lexed = [(Token.Text, content)]
     else:
         lexed = lex(content, lexer)
 
     for token_type, token_text in lexed:
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens, content):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     token_buffer = []
     for token_class, token_text in tokens:
         parts = token_text.split('\n')
         for part in parts[:-1]:
             token_buffer.append((token_class, part))
             yield token_buffer
             token_buffer = []
 
         token_buffer.append((token_class, parts[-1]))
 
     if token_buffer:
         yield token_buffer
     elif content:
         # this is a special case, we have the content, but tokenization didn't produce
         # any results. THis can happen if know file extensions like .css have some bogus
         # unicode content without any newline characters
         yield [(pygment_token_class(Token.Text), content)]
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
-            result.append(u'<span class="%s">' % token_class)
+            result.append('<span class="%s">' % token_class)
         else:
-            result.append(u'<span>')
+            result.append('<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
-                result.append(u'<%s>' % op_tag)
+                result.append('<%s>' % op_tag)
 
             # NOTE(marcink): in some cases of mixed encodings, we might run into
             # troubles in the html_escape, in this case we say unicode force on token_text
             # that would ensure "correct" data even with the cost of rendered
             try:
                 escaped_text = html_escape(token_text)
             except TypeError:
                 escaped_text = html_escape(safe_unicode(token_text))
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
-                result.append(u'</%s>' % op_tag)
+                result.append('</%s>' % op_tag)
 
-        result.append(u'</span>')
+        result.append('</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
     for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
         ops = []
         for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
     similarity = difflib.SequenceMatcher(None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
     if similarity < 0.6: # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
     token_sequence_matcher = difflib.SequenceMatcher(None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
         # check the differences by token block types first to give a more
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, 'del', token_text))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, 'ins', token_text))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
                 for char in token_text:
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
                 for char in token_text:
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
                 token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
                 token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11  # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], 'del', c))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
                             nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 def diffset_node_getter(commit):
     def get_node(fname):
         try:
             return commit.get_node(fname)
         except NodeDoesNotExistError:
             return None
 
     return get_node
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL'  # highlights using original file, slow
     HL_FAST = 'FAST'  # highlights using just the line, fast but not correct
                       # in the case of multiline code
     HL_NONE = 'NONE'  # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_repo_name=None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  # files over this size will use fast highlighting
                  max_file_size_limit=150 * 1024,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {
             'before': {},
             'after': {}
         }
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.target_repo_name = target_repo_name or repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             target_repo_name=self.target_repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = StrictAttributeDict(dict(
                 source_ref=diffset.source_ref,
                 target_ref=diffset.target_ref,
                 repo_name=diffset.repo_name,
                 source_repo_name=diffset.source_repo_name,
                 target_repo_name=diffset.target_repo_name,
             ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r', patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             node_hl_mode = self.HL_NONE if patch['chunks'] == [] else None
             hl_mode = node_hl_mode or self.highlight_mode
 
             if hl_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif hl_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
         raw_id_uid = ''
         if self.source_nodes.get(source_filename):
             raw_id_uid = self.source_nodes[source_filename].commit.raw_id
 
         if not raw_id_uid and self.target_nodes.get(target_filename):
             # in case this is a new file we only have it in target
             raw_id_uid = self.target_nodes[target_filename].commit.raw_id
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': patch['is_limited_diff'],
             'hunks': [],
             'hunk_ops': None,
             'diffset': self,
             'raw_id': raw_id_uid,
         })
 
         file_chunks = patch['chunks'][1:]
         for i, hunk in enumerate(file_chunks, 1):
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             hunkbit.index = i
             filediff.hunks.append(hunkbit)
 
         # Simulate hunk on OPS type line which doesn't really contain any diff
         # this allows commenting on those
         if not file_chunks:
             actions = []
             for op_id, op_text in filediff.patch['stats']['ops'].items():
                 if op_id == DEL_FILENODE:
-                    actions.append(u'file was removed')
+                    actions.append('file was removed')
                 elif op_id == BIN_FILENODE:
-                    actions.append(u'binary diff hidden')
+                    actions.append('binary diff hidden')
                 else:
                     actions.append(safe_unicode(op_text))
-            action_line = u'NO CONTENT: ' + \
-                          u', '.join(actions) or u'UNDEFINED_ACTION'
+            action_line = 'NO CONTENT: ' + \
+                          ', '.join(actions) or 'UNDEFINED_ACTION'
 
             hunk_ops = {'source_length': 0, 'source_start': 0,
                         'lines': [
                             {'new_lineno': 0, 'old_lineno': 1,
                              'action': 'unmod-no-hl', 'line': action_line}
                         ],
-                        'section_header': u'', 'target_start': 1, 'target_length': 1}
+                        'section_header': '', 'target_start': 1, 'target_length': 1}
 
             hunkbit = self.parse_hunk(hunk_ops, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunk_ops = hunkbit
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
             if line['action'] in ['unmod', 'unmod-no-hl']:
                 no_hl = line['action'] == 'unmod-no-hl'
                 result.lines.extend(
                     self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
             elif line['action'] == 'new-no-nl':
                 after.append(line)
 
         all_actions = [x['action'] for x in after] + [x['action'] for x in before]
         no_hl = {x for x in all_actions} == {'unmod-no-hl'}
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
         # NOTE(marcink): we must keep list() call here so we can cache the result...
         result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file,
                     no_hl=False):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
 
         before_newline = AttributeDict()
         after_newline = AttributeDict()
         if before_lines and before_lines[-1]['action'] == 'old-no-nl':
             before_newline_line = before_lines.pop(-1)
             before_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', before_newline_line['line'])]]))
 
         if after_lines and after_lines[-1]['action'] == 'new-no-nl':
             after_newline_line = after_lines.pop(-1)
             after_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', after_newline_line['line'])]]))
 
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
                     before_tokens = [('nonl', before['line'])]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'], line_number=before['old_lineno'],
                         input_file=source_file, no_hl=no_hl, source='before')
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
 
                 original.get_comment_args = (
                     source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
                     after_tokens = [('nonl', after['line'])]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         input_file=target_file, no_hl=no_hl, source='after')
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
 
                 modified.get_comment_args = (target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
             if not before_lines and before_newline:
                 original.content += before_newline.content
                 before_newline = None
             if not after_lines and after_newline:
                 modified.content += after_newline.content
                 after_newline = None
 
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_line_tokens(self, line_text, line_number, input_file=None, no_hl=False, source=''):
         filenode = None
         filename = None
 
         if isinstance(input_file, str):
             filename = input_file
         elif isinstance(input_file, FileNode):
             filenode = input_file
             filename = input_file.unicode_path
 
         hl_mode = self.HL_NONE if no_hl else self.highlight_mode
         if hl_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
             file_size_allowed = input_file.size < self.max_file_size_limit
             if line_number and file_size_allowed:
                 return self.get_tokenized_filenode_line(input_file, line_number, lexer, source)
 
         if hl_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None, source=''):
 
         def tokenize(_filenode):
             self.highlighted_filenodes[source][filenode] = filenode_as_lines_tokens(filenode, lexer)
 
         if filenode not in self.highlighted_filenodes[source]:
             tokenize(filenode)
 
         try:
             return self.highlighted_filenodes[source][filenode][line_number - 1]
         except Exception:
             log.exception('diff rendering error')
-            return [('', u'L{}: rhodecode diff rendering error'.format(line_number))]
+            return [('', 'L{}: rhodecode diff rendering error'.format(line_number))]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'unmod-no-hl': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.get_comment_args)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/lib/db_manage.py

@@ -1,680 +1,680 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print(('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper())
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args=None):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def db_exists(self):
         if not self.sa:
             self.init_db()
         try:
             self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
             return True
         except Exception:
             return False
         finally:
             self.sa.rollback()
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('db tables bootstrap: Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def run_post_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         if version:
             DbMigrateVersion.set_version(version)
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database db_uri under version '
                    'control with version {}'.format(curr_version))
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to upgrade database from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         self.run_post_migration_tasks()
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == User.DEFAULT_USER)\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
-            username = input('Specify admin username:')
+            username = eval(input('Specify admin username:'))
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
-            email = input('Specify admin email:')
+            email = eval(input('Specify admin email:'))
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution.exchange'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'topic'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
         defaults = [
             ('auth_plugins',
              'egg:rhodecode-enterprise-ce#token,egg:rhodecode-enterprise-ce#rhodecode',
              'list'),
 
             ('auth_authtoken_enabled',
              'True',
              'bool'),
 
             ('auth_rhodecode_enabled',
              'True',
              'bool'),
         ]
         for k, v, t in defaults:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
-            path = input(
+            path = eval(input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
-            )
+            ))
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
 
             q = ('Given path %s is not writeable, do you want to '
                  'continue with read only mode ? [y/n]' % (path,))
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = ('Path looks like a symlink, RhodeCode Enterprise will store '
                  'given path as %s ? [y/n]') % (real_path,)
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
 
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
             ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
 
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
 
             # VCS Settings
             ('pr_merge_enabled', True, 'bool'),
             ('use_outdated_comments', True, 'bool'),
             ('diff_cache', True, 'bool'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`', username)
         user = UserModel().create_or_update(
-            username, password, email, firstname=u'RhodeCode', lastname=u'Admin',
+            username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
-                description=u'BUILTIN TOKEN')
+                description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
-                                            firstname=u'Anonymous',
-                                            lastname=u'User',
+                                            firstname='Anonymous',
+                                            lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
-        # controlls anonymous access
+        # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/diff_match_patch.py

@@ -1,2031 +1,2031 b''
 """Diff Match and Patch
 
 Copyright 2006 Google Inc.
 http://code.google.com/p/google-diff-match-patch/
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
 
 """Functions for diff, match and patch.
 
 Computes the difference between two texts to create a patch.
 Applies the patch onto another text, allowing for errors.
 """
 
 __author__ = "[e-mail] (Neil Fraser)"
 
 import math
 import re
 import sys
 import time
 import urllib.request, urllib.parse, urllib.error
 
 
 class diff_match_patch:
     """Class containing the diff, match and patch methods.
 
     Also contains the behaviour settings.
     """
 
     def __init__(self):
         """Inits a diff_match_patch object with default settings.
         Redefine these in your program to override the defaults.
         """
 
         # Number of seconds to map a diff before giving up (0 for infinity).
         self.Diff_Timeout = 1.0
         # Cost of an empty edit operation in terms of edit characters.
         self.Diff_EditCost = 4
         # At what point is no match declared (0.0 = perfection, 1.0 = very loose).
         self.Match_Threshold = 0.5
         # How far to search for a match (0 = exact location, 1000+ = broad match).
         # A match this many characters away from the expected location will add
         # 1.0 to the score (0.0 is a perfect match).
         self.Match_Distance = 1000
         # When deleting a large block of text (over ~64 characters), how close do
         # the contents have to be to match the expected contents. (0.0 = perfection,
         # 1.0 = very loose).  Note that Match_Threshold controls how closely the
         # end points of a delete need to match.
         self.Patch_DeleteThreshold = 0.5
         # Chunk size for context length.
         self.Patch_Margin = 4
 
         # The number of bits in an int.
         # Python has no maximum, thus to disable patch splitting set to 0.
         # However to avoid long patches in certain pathological cases, use 32.
         # Multiple short patches (using native ints) are much faster than long ones.
         self.Match_MaxBits = 32
 
     #  DIFF FUNCTIONS
 
     # The data structure representing a diff is an array of tuples:
     # [(DIFF_DELETE, "Hello"), (DIFF_INSERT, "Goodbye"), (DIFF_EQUAL, " world.")]
     # which means: delete "Hello", add "Goodbye" and keep " world."
     DIFF_DELETE = -1
     DIFF_INSERT = 1
     DIFF_EQUAL = 0
 
     def diff_main(self, text1, text2, checklines=True, deadline=None):
         """Find the differences between two texts.  Simplifies the problem by
           stripping any common prefix or suffix off the texts before diffing.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           checklines: Optional speedup flag.  If present and false, then don't run
             a line-level diff first to identify the changed areas.
             Defaults to true, which does a faster, slightly less optimal diff.
           deadline: Optional time when the diff should be complete by.  Used
             internally for recursive calls.  Users should set DiffTimeout instead.
 
         Returns:
           Array of changes.
         """
         # Set a deadline by which time the diff must be complete.
         if deadline is None:
             # Unlike in most languages, Python counts time in seconds.
             if self.Diff_Timeout <= 0:
                 deadline = sys.maxsize
             else:
                 deadline = time.time() + self.Diff_Timeout
 
         # Check for null inputs.
         if text1 is None or text2 is None:
             raise ValueError("Null inputs. (diff_main)")
 
         # Check for equality (speedup).
         if text1 == text2:
             if text1:
                 return [(self.DIFF_EQUAL, text1)]
             return []
 
         # Trim off common prefix (speedup).
         commonlength = self.diff_commonPrefix(text1, text2)
         commonprefix = text1[:commonlength]
         text1 = text1[commonlength:]
         text2 = text2[commonlength:]
 
         # Trim off common suffix (speedup).
         commonlength = self.diff_commonSuffix(text1, text2)
         if commonlength == 0:
             commonsuffix = ""
         else:
             commonsuffix = text1[-commonlength:]
             text1 = text1[:-commonlength]
             text2 = text2[:-commonlength]
 
         # Compute the diff on the middle block.
         diffs = self.diff_compute(text1, text2, checklines, deadline)
 
         # Restore the prefix and suffix.
         if commonprefix:
             diffs[:0] = [(self.DIFF_EQUAL, commonprefix)]
         if commonsuffix:
             diffs.append((self.DIFF_EQUAL, commonsuffix))
         self.diff_cleanupMerge(diffs)
         return diffs
 
     def diff_compute(self, text1, text2, checklines, deadline):
         """Find the differences between two texts.  Assumes that the texts do not
           have any common prefix or suffix.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           checklines: Speedup flag.  If false, then don't run a line-level diff
             first to identify the changed areas.
             If true, then run a faster, slightly less optimal diff.
           deadline: Time when the diff should be complete by.
 
         Returns:
           Array of changes.
         """
         if not text1:
             # Just add some text (speedup).
             return [(self.DIFF_INSERT, text2)]
 
         if not text2:
             # Just delete some text (speedup).
             return [(self.DIFF_DELETE, text1)]
 
         if len(text1) > len(text2):
             (longtext, shorttext) = (text1, text2)
         else:
             (shorttext, longtext) = (text1, text2)
         i = longtext.find(shorttext)
         if i != -1:
             # Shorter text is inside the longer text (speedup).
             diffs = [
                 (self.DIFF_INSERT, longtext[:i]),
                 (self.DIFF_EQUAL, shorttext),
                 (self.DIFF_INSERT, longtext[i + len(shorttext) :]),
             ]
             # Swap insertions for deletions if diff is reversed.
             if len(text1) > len(text2):
                 diffs[0] = (self.DIFF_DELETE, diffs[0][1])
                 diffs[2] = (self.DIFF_DELETE, diffs[2][1])
             return diffs
 
         if len(shorttext) == 1:
             # Single character string.
             # After the previous speedup, the character can't be an equality.
             return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
         # Check to see if the problem can be split in two.
         hm = self.diff_halfMatch(text1, text2)
         if hm:
             # A half-match was found, sort out the return data.
             (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
             # Send both pairs off for separate processing.
             diffs_a = self.diff_main(text1_a, text2_a, checklines, deadline)
             diffs_b = self.diff_main(text1_b, text2_b, checklines, deadline)
             # Merge the results.
             return diffs_a + [(self.DIFF_EQUAL, mid_common)] + diffs_b
 
         if checklines and len(text1) > 100 and len(text2) > 100:
             return self.diff_lineMode(text1, text2, deadline)
 
         return self.diff_bisect(text1, text2, deadline)
 
     def diff_lineMode(self, text1, text2, deadline):
         """Do a quick line-level diff on both strings, then rediff the parts for
           greater accuracy.
           This speedup can produce non-minimal diffs.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           deadline: Time when the diff should be complete by.
 
         Returns:
           Array of changes.
         """
 
         # Scan the text on a line-by-line basis first.
         (text1, text2, linearray) = self.diff_linesToChars(text1, text2)
 
         diffs = self.diff_main(text1, text2, False, deadline)
 
         # Convert the diff back to original text.
         self.diff_charsToLines(diffs, linearray)
         # Eliminate freak matches (e.g. blank lines)
         self.diff_cleanupSemantic(diffs)
 
         # Rediff any replacement blocks, this time character-by-character.
         # Add a dummy entry at the end.
         diffs.append((self.DIFF_EQUAL, ""))
         pointer = 0
         count_delete = 0
         count_insert = 0
         text_delete = ""
         text_insert = ""
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_INSERT:
                 count_insert += 1
                 text_insert += diffs[pointer][1]
             elif diffs[pointer][0] == self.DIFF_DELETE:
                 count_delete += 1
                 text_delete += diffs[pointer][1]
             elif diffs[pointer][0] == self.DIFF_EQUAL:
                 # Upon reaching an equality, check for prior redundancies.
                 if count_delete >= 1 and count_insert >= 1:
                     # Delete the offending records and add the merged ones.
                     a = self.diff_main(text_delete, text_insert, False, deadline)
                     diffs[pointer - count_delete - count_insert : pointer] = a
                     pointer = pointer - count_delete - count_insert + len(a)
                 count_insert = 0
                 count_delete = 0
                 text_delete = ""
                 text_insert = ""
 
             pointer += 1
 
         diffs.pop()  # Remove the dummy entry at the end.
 
         return diffs
 
     def diff_bisect(self, text1, text2, deadline):
         """Find the 'middle snake' of a diff, split the problem in two
           and return the recursively constructed diff.
           See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           deadline: Time at which to bail if not yet complete.
 
         Returns:
           Array of diff tuples.
         """
 
         # Cache the text lengths to prevent multiple calls.
         text1_length = len(text1)
         text2_length = len(text2)
         max_d = (text1_length + text2_length + 1) // 2
         v_offset = max_d
         v_length = 2 * max_d
         v1 = [-1] * v_length
         v1[v_offset + 1] = 0
         v2 = v1[:]
         delta = text1_length - text2_length
         # If the total number of characters is odd, then the front path will
         # collide with the reverse path.
         front = delta % 2 != 0
         # Offsets for start and end of k loop.
         # Prevents mapping of space beyond the grid.
         k1start = 0
         k1end = 0
         k2start = 0
         k2end = 0
         for d in range(max_d):
             # Bail out if deadline is reached.
             if time.time() > deadline:
                 break
 
             # Walk the front path one step.
             for k1 in range(-d + k1start, d + 1 - k1end, 2):
                 k1_offset = v_offset + k1
                 if k1 == -d or (k1 != d and v1[k1_offset - 1] < v1[k1_offset + 1]):
                     x1 = v1[k1_offset + 1]
                 else:
                     x1 = v1[k1_offset - 1] + 1
                 y1 = x1 - k1
                 while (
                     x1 < text1_length and y1 < text2_length and text1[x1] == text2[y1]
                 ):
                     x1 += 1
                     y1 += 1
                 v1[k1_offset] = x1
                 if x1 > text1_length:
                     # Ran off the right of the graph.
                     k1end += 2
                 elif y1 > text2_length:
                     # Ran off the bottom of the graph.
                     k1start += 2
                 elif front:
                     k2_offset = v_offset + delta - k1
                     if k2_offset >= 0 and k2_offset < v_length and v2[k2_offset] != -1:
                         # Mirror x2 onto top-left coordinate system.
                         x2 = text1_length - v2[k2_offset]
                         if x1 >= x2:
                             # Overlap detected.
                             return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
             # Walk the reverse path one step.
             for k2 in range(-d + k2start, d + 1 - k2end, 2):
                 k2_offset = v_offset + k2
                 if k2 == -d or (k2 != d and v2[k2_offset - 1] < v2[k2_offset + 1]):
                     x2 = v2[k2_offset + 1]
                 else:
                     x2 = v2[k2_offset - 1] + 1
                 y2 = x2 - k2
                 while (
                     x2 < text1_length
                     and y2 < text2_length
                     and text1[-x2 - 1] == text2[-y2 - 1]
                 ):
                     x2 += 1
                     y2 += 1
                 v2[k2_offset] = x2
                 if x2 > text1_length:
                     # Ran off the left of the graph.
                     k2end += 2
                 elif y2 > text2_length:
                     # Ran off the top of the graph.
                     k2start += 2
                 elif not front:
                     k1_offset = v_offset + delta - k2
                     if k1_offset >= 0 and k1_offset < v_length and v1[k1_offset] != -1:
                         x1 = v1[k1_offset]
                         y1 = v_offset + x1 - k1_offset
                         # Mirror x2 onto top-left coordinate system.
                         x2 = text1_length - x2
                         if x1 >= x2:
                             # Overlap detected.
                             return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
         # Diff took too long and hit the deadline or
         # number of diffs equals number of characters, no commonality at all.
         return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
     def diff_bisectSplit(self, text1, text2, x, y, deadline):
         """Given the location of the 'middle snake', split the diff in two parts
         and recurse.
 
         Args:
           text1: Old string to be diffed.
           text2: New string to be diffed.
           x: Index of split point in text1.
           y: Index of split point in text2.
           deadline: Time at which to bail if not yet complete.
 
         Returns:
           Array of diff tuples.
         """
         text1a = text1[:x]
         text2a = text2[:y]
         text1b = text1[x:]
         text2b = text2[y:]
 
         # Compute both diffs serially.
         diffs = self.diff_main(text1a, text2a, False, deadline)
         diffsb = self.diff_main(text1b, text2b, False, deadline)
 
         return diffs + diffsb
 
     def diff_linesToChars(self, text1, text2):
         """Split two texts into an array of strings.  Reduce the texts to a string
         of hashes where each Unicode character represents one line.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           Three element tuple, containing the encoded text1, the encoded text2 and
           the array of unique strings.  The zeroth element of the array of unique
           strings is intentionally blank.
         """
         lineArray = []  # e.g. lineArray[4] == "Hello\n"
         lineHash = {}  # e.g. lineHash["Hello\n"] == 4
 
         # "\x00" is a valid character, but various debuggers don't like it.
         # So we'll insert a junk entry to avoid generating a null character.
         lineArray.append("")
 
         def diff_linesToCharsMunge(text):
             """Split a text into an array of strings.  Reduce the texts to a string
             of hashes where each Unicode character represents one line.
             Modifies linearray and linehash through being a closure.
 
             Args:
               text: String to encode.
 
             Returns:
               Encoded string.
             """
             chars = []
             # Walk the text, pulling out a substring for each line.
             # text.split('\n') would would temporarily double our memory footprint.
             # Modifying text would create many large strings to garbage collect.
             lineStart = 0
             lineEnd = -1
             while lineEnd < len(text) - 1:
                 lineEnd = text.find("\n", lineStart)
                 if lineEnd == -1:
                     lineEnd = len(text) - 1
                 line = text[lineStart : lineEnd + 1]
                 lineStart = lineEnd + 1
 
                 if line in lineHash:
                     chars.append(chr(lineHash[line]))
                 else:
                     lineArray.append(line)
                     lineHash[line] = len(lineArray) - 1
                     chars.append(chr(len(lineArray) - 1))
             return "".join(chars)
 
         chars1 = diff_linesToCharsMunge(text1)
         chars2 = diff_linesToCharsMunge(text2)
         return (chars1, chars2, lineArray)
 
     def diff_charsToLines(self, diffs, lineArray):
         """Rehydrate the text in a diff from a string of line hashes to real lines
         of text.
 
         Args:
           diffs: Array of diff tuples.
           lineArray: Array of unique strings.
         """
         for x in range(len(diffs)):
             text = []
             for char in diffs[x][1]:
                 text.append(lineArray[ord(char)])
             diffs[x] = (diffs[x][0], "".join(text))
 
     def diff_commonPrefix(self, text1, text2):
         """Determine the common prefix of two strings.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           The number of characters common to the start of each string.
         """
         # Quick check for common null cases.
         if not text1 or not text2 or text1[0] != text2[0]:
             return 0
         # Binary search.
         # Performance analysis: http://neil.fraser.name/news/2007/10/09/
         pointermin = 0
         pointermax = min(len(text1), len(text2))
         pointermid = pointermax
         pointerstart = 0
         while pointermin < pointermid:
             if text1[pointerstart:pointermid] == text2[pointerstart:pointermid]:
                 pointermin = pointermid
                 pointerstart = pointermin
             else:
                 pointermax = pointermid
             pointermid = (pointermax - pointermin) // 2 + pointermin
         return pointermid
 
     def diff_commonSuffix(self, text1, text2):
         """Determine the common suffix of two strings.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           The number of characters common to the end of each string.
         """
         # Quick check for common null cases.
         if not text1 or not text2 or text1[-1] != text2[-1]:
             return 0
         # Binary search.
         # Performance analysis: http://neil.fraser.name/news/2007/10/09/
         pointermin = 0
         pointermax = min(len(text1), len(text2))
         pointermid = pointermax
         pointerend = 0
         while pointermin < pointermid:
             if (
                 text1[-pointermid : len(text1) - pointerend]
                 == text2[-pointermid : len(text2) - pointerend]
             ):
                 pointermin = pointermid
                 pointerend = pointermin
             else:
                 pointermax = pointermid
             pointermid = (pointermax - pointermin) // 2 + pointermin
         return pointermid
 
     def diff_commonOverlap(self, text1, text2):
         """Determine if the suffix of one string is the prefix of another.
 
         Args:
           text1 First string.
           text2 Second string.
 
         Returns:
           The number of characters common to the end of the first
           string and the start of the second string.
         """
         # Cache the text lengths to prevent multiple calls.
         text1_length = len(text1)
         text2_length = len(text2)
         # Eliminate the null case.
         if text1_length == 0 or text2_length == 0:
             return 0
         # Truncate the longer string.
         if text1_length > text2_length:
             text1 = text1[-text2_length:]
         elif text1_length < text2_length:
             text2 = text2[:text1_length]
         text_length = min(text1_length, text2_length)
         # Quick check for the worst case.
         if text1 == text2:
             return text_length
 
         # Start by looking for a single character match
         # and increase length until no match is found.
         # Performance analysis: http://neil.fraser.name/news/2010/11/04/
         best = 0
         length = 1
         while True:
             pattern = text1[-length:]
             found = text2.find(pattern)
             if found == -1:
                 return best
             length += found
             if found == 0 or text1[-length:] == text2[:length]:
                 best = length
                 length += 1
 
     def diff_halfMatch(self, text1, text2):
         """Do the two texts share a substring which is at least half the length of
         the longer text?
         This speedup can produce non-minimal diffs.
 
         Args:
           text1: First string.
           text2: Second string.
 
         Returns:
           Five element Array, containing the prefix of text1, the suffix of text1,
           the prefix of text2, the suffix of text2 and the common middle.  Or None
           if there was no match.
         """
         if self.Diff_Timeout <= 0:
             # Don't risk returning a non-optimal diff if we have unlimited time.
             return None
         if len(text1) > len(text2):
             (longtext, shorttext) = (text1, text2)
         else:
             (shorttext, longtext) = (text1, text2)
         if len(longtext) < 4 or len(shorttext) * 2 < len(longtext):
             return None  # Pointless.
 
         def diff_halfMatchI(longtext, shorttext, i):
             """Does a substring of shorttext exist within longtext such that the
             substring is at least half the length of longtext?
             Closure, but does not reference any external variables.
 
             Args:
               longtext: Longer string.
               shorttext: Shorter string.
               i: Start index of quarter length substring within longtext.
 
             Returns:
               Five element Array, containing the prefix of longtext, the suffix of
               longtext, the prefix of shorttext, the suffix of shorttext and the
               common middle.  Or None if there was no match.
             """
             seed = longtext[i : i + len(longtext) // 4]
             best_common = ""
             j = shorttext.find(seed)
             while j != -1:
                 prefixLength = self.diff_commonPrefix(longtext[i:], shorttext[j:])
                 suffixLength = self.diff_commonSuffix(longtext[:i], shorttext[:j])
                 if len(best_common) < suffixLength + prefixLength:
                     best_common = (
                         shorttext[j - suffixLength : j]
                         + shorttext[j : j + prefixLength]
                     )
                     best_longtext_a = longtext[: i - suffixLength]
                     best_longtext_b = longtext[i + prefixLength :]
                     best_shorttext_a = shorttext[: j - suffixLength]
                     best_shorttext_b = shorttext[j + prefixLength :]
                 j = shorttext.find(seed, j + 1)
 
             if len(best_common) * 2 >= len(longtext):
                 return (
                     best_longtext_a,
                     best_longtext_b,
                     best_shorttext_a,
                     best_shorttext_b,
                     best_common,
                 )
             else:
                 return None
 
         # First check if the second quarter is the seed for a half-match.
         hm1 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 3) // 4)
         # Check again based on the third quarter.
         hm2 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 1) // 2)
         if not hm1 and not hm2:
             return None
         elif not hm2:
             hm = hm1
         elif not hm1:
             hm = hm2
         else:
             # Both matched.  Select the longest.
             if len(hm1[4]) > len(hm2[4]):
                 hm = hm1
             else:
                 hm = hm2
 
         # A half-match was found, sort out the return data.
         if len(text1) > len(text2):
             (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
         else:
             (text2_a, text2_b, text1_a, text1_b, mid_common) = hm
         return (text1_a, text1_b, text2_a, text2_b, mid_common)
 
     def diff_cleanupSemantic(self, diffs):
         """Reduce the number of edits by eliminating semantically trivial
         equalities.
 
         Args:
           diffs: Array of diff tuples.
         """
         changes = False
         equalities = []  # Stack of indices where equalities are found.
         lastequality = None  # Always equal to diffs[equalities[-1]][1]
         pointer = 0  # Index of current position.
         # Number of chars that changed prior to the equality.
         length_insertions1, length_deletions1 = 0, 0
         # Number of chars that changed after the equality.
         length_insertions2, length_deletions2 = 0, 0
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
                 equalities.append(pointer)
                 length_insertions1, length_insertions2 = length_insertions2, 0
                 length_deletions1, length_deletions2 = length_deletions2, 0
                 lastequality = diffs[pointer][1]
             else:  # An insertion or deletion.
                 if diffs[pointer][0] == self.DIFF_INSERT:
                     length_insertions2 += len(diffs[pointer][1])
                 else:
                     length_deletions2 += len(diffs[pointer][1])
                 # Eliminate an equality that is smaller or equal to the edits on both
                 # sides of it.
                 if (
                     lastequality
                     and (
                         len(lastequality) <= max(length_insertions1, length_deletions1)
                     )
                     and (
                         len(lastequality) <= max(length_insertions2, length_deletions2)
                     )
                 ):
                     # Duplicate record.
                     diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
                     # Change second copy to insert.
                     diffs[equalities[-1] + 1] = (
                         self.DIFF_INSERT,
                         diffs[equalities[-1] + 1][1],
                     )
                     # Throw away the equality we just deleted.
                     equalities.pop()
                     # Throw away the previous equality (it needs to be reevaluated).
                     if len(equalities):
                         equalities.pop()
                     if len(equalities):
                         pointer = equalities[-1]
                     else:
                         pointer = -1
                     # Reset the counters.
                     length_insertions1, length_deletions1 = 0, 0
                     length_insertions2, length_deletions2 = 0, 0
                     lastequality = None
                     changes = True
             pointer += 1
 
         # Normalize the diff.
         if changes:
             self.diff_cleanupMerge(diffs)
         self.diff_cleanupSemanticLossless(diffs)
 
         # Find any overlaps between deletions and insertions.
         # e.g: <del>abcxxx</del><ins>xxxdef</ins>
         #   -> <del>abc</del>xxx<ins>def</ins>
         # e.g: <del>xxxabc</del><ins>defxxx</ins>
         #   -> <ins>def</ins>xxx<del>abc</del>
         # Only extract an overlap if it is as big as the edit ahead or behind it.
         pointer = 1
         while pointer < len(diffs):
             if (
                 diffs[pointer - 1][0] == self.DIFF_DELETE
                 and diffs[pointer][0] == self.DIFF_INSERT
             ):
                 deletion = diffs[pointer - 1][1]
                 insertion = diffs[pointer][1]
                 overlap_length1 = self.diff_commonOverlap(deletion, insertion)
                 overlap_length2 = self.diff_commonOverlap(insertion, deletion)
                 if overlap_length1 >= overlap_length2:
                     if (
                         overlap_length1 >= len(deletion) / 2.0
                         or overlap_length1 >= len(insertion) / 2.0
                     ):
                         # Overlap found.  Insert an equality and trim the surrounding edits.
                         diffs.insert(
                             pointer, (self.DIFF_EQUAL, insertion[:overlap_length1])
                         )
                         diffs[pointer - 1] = (
                             self.DIFF_DELETE,
                             deletion[: len(deletion) - overlap_length1],
                         )
                         diffs[pointer + 1] = (
                             self.DIFF_INSERT,
                             insertion[overlap_length1:],
                         )
                         pointer += 1
                 else:
                     if (
                         overlap_length2 >= len(deletion) / 2.0
                         or overlap_length2 >= len(insertion) / 2.0
                     ):
                         # Reverse overlap found.
                         # Insert an equality and swap and trim the surrounding edits.
                         diffs.insert(
                             pointer, (self.DIFF_EQUAL, deletion[:overlap_length2])
                         )
                         diffs[pointer - 1] = (
                             self.DIFF_INSERT,
                             insertion[: len(insertion) - overlap_length2],
                         )
                         diffs[pointer + 1] = (
                             self.DIFF_DELETE,
                             deletion[overlap_length2:],
                         )
                         pointer += 1
                 pointer += 1
             pointer += 1
 
     def diff_cleanupSemanticLossless(self, diffs):
         """Look for single edits surrounded on both sides by equalities
         which can be shifted sideways to align the edit to a word boundary.
         e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.
 
         Args:
           diffs: Array of diff tuples.
         """
 
         def diff_cleanupSemanticScore(one, two):
             """Given two strings, compute a score representing whether the
             internal boundary falls on logical boundaries.
             Scores range from 6 (best) to 0 (worst).
             Closure, but does not reference any external variables.
 
             Args:
               one: First string.
               two: Second string.
 
             Returns:
               The score.
             """
             if not one or not two:
                 # Edges are the best.
                 return 6
 
             # Each port of this function behaves slightly differently due to
             # subtle differences in each language's definition of things like
             # 'whitespace'.  Since this function's purpose is largely cosmetic,
             # the choice has been made to use each language's native features
             # rather than force total conformity.
             char1 = one[-1]
             char2 = two[0]
             nonAlphaNumeric1 = not char1.isalnum()
             nonAlphaNumeric2 = not char2.isalnum()
             whitespace1 = nonAlphaNumeric1 and char1.isspace()
             whitespace2 = nonAlphaNumeric2 and char2.isspace()
             lineBreak1 = whitespace1 and (char1 == "\r" or char1 == "\n")
             lineBreak2 = whitespace2 and (char2 == "\r" or char2 == "\n")
             blankLine1 = lineBreak1 and self.BLANKLINEEND.search(one)
             blankLine2 = lineBreak2 and self.BLANKLINESTART.match(two)
 
             if blankLine1 or blankLine2:
                 # Five points for blank lines.
                 return 5
             elif lineBreak1 or lineBreak2:
                 # Four points for line breaks.
                 return 4
             elif nonAlphaNumeric1 and not whitespace1 and whitespace2:
                 # Three points for end of sentences.
                 return 3
             elif whitespace1 or whitespace2:
                 # Two points for whitespace.
                 return 2
             elif nonAlphaNumeric1 or nonAlphaNumeric2:
                 # One point for non-alphanumeric.
                 return 1
             return 0
 
         pointer = 1
         # Intentionally ignore the first and last element (don't need checking).
         while pointer < len(diffs) - 1:
             if (
                 diffs[pointer - 1][0] == self.DIFF_EQUAL
                 and diffs[pointer + 1][0] == self.DIFF_EQUAL
             ):
                 # This is a single edit surrounded by equalities.
                 equality1 = diffs[pointer - 1][1]
                 edit = diffs[pointer][1]
                 equality2 = diffs[pointer + 1][1]
 
                 # First, shift the edit as far left as possible.
                 commonOffset = self.diff_commonSuffix(equality1, edit)
                 if commonOffset:
                     commonString = edit[-commonOffset:]
                     equality1 = equality1[:-commonOffset]
                     edit = commonString + edit[:-commonOffset]
                     equality2 = commonString + equality2
 
                 # Second, step character by character right, looking for the best fit.
                 bestEquality1 = equality1
                 bestEdit = edit
                 bestEquality2 = equality2
                 bestScore = diff_cleanupSemanticScore(
                     equality1, edit
                 ) + diff_cleanupSemanticScore(edit, equality2)
                 while edit and equality2 and edit[0] == equality2[0]:
                     equality1 += edit[0]
                     edit = edit[1:] + equality2[0]
                     equality2 = equality2[1:]
                     score = diff_cleanupSemanticScore(
                         equality1, edit
                     ) + diff_cleanupSemanticScore(edit, equality2)
                     # The >= encourages trailing rather than leading whitespace on edits.
                     if score >= bestScore:
                         bestScore = score
                         bestEquality1 = equality1
                         bestEdit = edit
                         bestEquality2 = equality2
 
                 if diffs[pointer - 1][1] != bestEquality1:
                     # We have an improvement, save it back to the diff.
                     if bestEquality1:
                         diffs[pointer - 1] = (diffs[pointer - 1][0], bestEquality1)
                     else:
                         del diffs[pointer - 1]
                         pointer -= 1
                     diffs[pointer] = (diffs[pointer][0], bestEdit)
                     if bestEquality2:
                         diffs[pointer + 1] = (diffs[pointer + 1][0], bestEquality2)
                     else:
                         del diffs[pointer + 1]
                         pointer -= 1
             pointer += 1
 
     # Define some regex patterns for matching boundaries.
     BLANKLINEEND = re.compile(r"\n\r?\n$")
     BLANKLINESTART = re.compile(r"^\r?\n\r?\n")
 
     def diff_cleanupEfficiency(self, diffs):
         """Reduce the number of edits by eliminating operationally trivial
         equalities.
 
         Args:
           diffs: Array of diff tuples.
         """
         changes = False
         equalities = []  # Stack of indices where equalities are found.
         lastequality = None  # Always equal to diffs[equalities[-1]][1]
         pointer = 0  # Index of current position.
         pre_ins = False  # Is there an insertion operation before the last equality.
         pre_del = False  # Is there a deletion operation before the last equality.
         post_ins = False  # Is there an insertion operation after the last equality.
         post_del = False  # Is there a deletion operation after the last equality.
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
                 if len(diffs[pointer][1]) < self.Diff_EditCost and (
                     post_ins or post_del
                 ):
                     # Candidate found.
                     equalities.append(pointer)
                     pre_ins = post_ins
                     pre_del = post_del
                     lastequality = diffs[pointer][1]
                 else:
                     # Not a candidate, and can never become one.
                     equalities = []
                     lastequality = None
 
                 post_ins = post_del = False
             else:  # An insertion or deletion.
                 if diffs[pointer][0] == self.DIFF_DELETE:
                     post_del = True
                 else:
                     post_ins = True
 
                 # Five types to be split:
                 # <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
                 # <ins>A</ins>X<ins>C</ins><del>D</del>
                 # <ins>A</ins><del>B</del>X<ins>C</ins>
                 # <ins>A</del>X<ins>C</ins><del>D</del>
                 # <ins>A</ins><del>B</del>X<del>C</del>
 
                 if lastequality and (
                     (pre_ins and pre_del and post_ins and post_del)
                     or (
                         (len(lastequality) < self.Diff_EditCost / 2)
                         and (pre_ins + pre_del + post_ins + post_del) == 3
                     )
                 ):
                     # Duplicate record.
                     diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
                     # Change second copy to insert.
                     diffs[equalities[-1] + 1] = (
                         self.DIFF_INSERT,
                         diffs[equalities[-1] + 1][1],
                     )
                     equalities.pop()  # Throw away the equality we just deleted.
                     lastequality = None
                     if pre_ins and pre_del:
                         # No changes made which could affect previous entry, keep going.
                         post_ins = post_del = True
                         equalities = []
                     else:
                         if len(equalities):
                             equalities.pop()  # Throw away the previous equality.
                         if len(equalities):
                             pointer = equalities[-1]
                         else:
                             pointer = -1
                         post_ins = post_del = False
                     changes = True
             pointer += 1
 
         if changes:
             self.diff_cleanupMerge(diffs)
 
     def diff_cleanupMerge(self, diffs):
         """Reorder and merge like edit sections.  Merge equalities.
         Any edit section can move as long as it doesn't cross an equality.
 
         Args:
           diffs: Array of diff tuples.
         """
         diffs.append((self.DIFF_EQUAL, ""))  # Add a dummy entry at the end.
         pointer = 0
         count_delete = 0
         count_insert = 0
         text_delete = ""
         text_insert = ""
         while pointer < len(diffs):
             if diffs[pointer][0] == self.DIFF_INSERT:
                 count_insert += 1
                 text_insert += diffs[pointer][1]
                 pointer += 1
             elif diffs[pointer][0] == self.DIFF_DELETE:
                 count_delete += 1
                 text_delete += diffs[pointer][1]
                 pointer += 1
             elif diffs[pointer][0] == self.DIFF_EQUAL:
                 # Upon reaching an equality, check for prior redundancies.
                 if count_delete + count_insert > 1:
                     if count_delete != 0 and count_insert != 0:
                         # Factor out any common prefixies.
                         commonlength = self.diff_commonPrefix(text_insert, text_delete)
                         if commonlength != 0:
                             x = pointer - count_delete - count_insert - 1
                             if x >= 0 and diffs[x][0] == self.DIFF_EQUAL:
                                 diffs[x] = (
                                     diffs[x][0],
                                     diffs[x][1] + text_insert[:commonlength],
                                 )
                             else:
                                 diffs.insert(
                                     0, (self.DIFF_EQUAL, text_insert[:commonlength])
                                 )
                                 pointer += 1
                             text_insert = text_insert[commonlength:]
                             text_delete = text_delete[commonlength:]
                         # Factor out any common suffixies.
                         commonlength = self.diff_commonSuffix(text_insert, text_delete)
                         if commonlength != 0:
                             diffs[pointer] = (
                                 diffs[pointer][0],
                                 text_insert[-commonlength:] + diffs[pointer][1],
                             )
                             text_insert = text_insert[:-commonlength]
                             text_delete = text_delete[:-commonlength]
                     # Delete the offending records and add the merged ones.
                     if count_delete == 0:
                         diffs[pointer - count_insert : pointer] = [
                             (self.DIFF_INSERT, text_insert)
                         ]
                     elif count_insert == 0:
                         diffs[pointer - count_delete : pointer] = [
                             (self.DIFF_DELETE, text_delete)
                         ]
                     else:
                         diffs[pointer - count_delete - count_insert : pointer] = [
                             (self.DIFF_DELETE, text_delete),
                             (self.DIFF_INSERT, text_insert),
                         ]
                     pointer = pointer - count_delete - count_insert + 1
                     if count_delete != 0:
                         pointer += 1
                     if count_insert != 0:
                         pointer += 1
                 elif pointer != 0 and diffs[pointer - 1][0] == self.DIFF_EQUAL:
                     # Merge this equality with the previous one.
                     diffs[pointer - 1] = (
                         diffs[pointer - 1][0],
                         diffs[pointer - 1][1] + diffs[pointer][1],
                     )
                     del diffs[pointer]
                 else:
                     pointer += 1
 
                 count_insert = 0
                 count_delete = 0
                 text_delete = ""
                 text_insert = ""
 
         if diffs[-1][1] == "":
             diffs.pop()  # Remove the dummy entry at the end.
 
         # Second pass: look for single edits surrounded on both sides by equalities
         # which can be shifted sideways to eliminate an equality.
         # e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
         changes = False
         pointer = 1
         # Intentionally ignore the first and last element (don't need checking).
         while pointer < len(diffs) - 1:
             if (
                 diffs[pointer - 1][0] == self.DIFF_EQUAL
                 and diffs[pointer + 1][0] == self.DIFF_EQUAL
             ):
                 # This is a single edit surrounded by equalities.
                 if diffs[pointer][1].endswith(diffs[pointer - 1][1]):
                     # Shift the edit over the previous equality.
                     diffs[pointer] = (
                         diffs[pointer][0],
                         diffs[pointer - 1][1]
                         + diffs[pointer][1][: -len(diffs[pointer - 1][1])],
                     )
                     diffs[pointer + 1] = (
                         diffs[pointer + 1][0],
                         diffs[pointer - 1][1] + diffs[pointer + 1][1],
                     )
                     del diffs[pointer - 1]
                     changes = True
                 elif diffs[pointer][1].startswith(diffs[pointer + 1][1]):
                     # Shift the edit over the next equality.
                     diffs[pointer - 1] = (
                         diffs[pointer - 1][0],
                         diffs[pointer - 1][1] + diffs[pointer + 1][1],
                     )
                     diffs[pointer] = (
                         diffs[pointer][0],
                         diffs[pointer][1][len(diffs[pointer + 1][1]) :]
                         + diffs[pointer + 1][1],
                     )
                     del diffs[pointer + 1]
                     changes = True
             pointer += 1
 
         # If shifts were made, the diff needs reordering and another shift sweep.
         if changes:
             self.diff_cleanupMerge(diffs)
 
     def diff_xIndex(self, diffs, loc):
         """loc is a location in text1, compute and return the equivalent location
         in text2.  e.g. "The cat" vs "The big cat", 1->1, 5->8
 
         Args:
           diffs: Array of diff tuples.
           loc: Location within text1.
 
         Returns:
           Location within text2.
         """
         chars1 = 0
         chars2 = 0
         last_chars1 = 0
         last_chars2 = 0
         for x in range(len(diffs)):
             (op, text) = diffs[x]
             if op != self.DIFF_INSERT:  # Equality or deletion.
                 chars1 += len(text)
             if op != self.DIFF_DELETE:  # Equality or insertion.
                 chars2 += len(text)
             if chars1 > loc:  # Overshot the location.
                 break
             last_chars1 = chars1
             last_chars2 = chars2
 
         if len(diffs) != x and diffs[x][0] == self.DIFF_DELETE:
             # The location was deleted.
             return last_chars2
         # Add the remaining len(character).
         return last_chars2 + (loc - last_chars1)
 
     def diff_prettyHtml(self, diffs):
         """Convert a diff array into a pretty HTML report.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           HTML representation.
         """
         html = []
         for op, data in diffs:
             text = (
                 data.replace("&", "&amp;")
                 .replace("<", "&lt;")
                 .replace(">", "&gt;")
                 .replace("\n", "&para;<br>")
             )
             if op == self.DIFF_INSERT:
                 html.append('<ins style="background:#e6ffe6;">%s</ins>' % text)
             elif op == self.DIFF_DELETE:
                 html.append('<del style="background:#ffe6e6;">%s</del>' % text)
             elif op == self.DIFF_EQUAL:
                 html.append("<span>%s</span>" % text)
         return "".join(html)
 
     def diff_text1(self, diffs):
         """Compute and return the source text (all equalities and deletions).
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Source text.
         """
         text = []
         for op, data in diffs:
             if op != self.DIFF_INSERT:
                 text.append(data)
         return "".join(text)
 
     def diff_text2(self, diffs):
         """Compute and return the destination text (all equalities and insertions).
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Destination text.
         """
         text = []
         for op, data in diffs:
             if op != self.DIFF_DELETE:
                 text.append(data)
         return "".join(text)
 
     def diff_levenshtein(self, diffs):
         """Compute the Levenshtein distance; the number of inserted, deleted or
         substituted characters.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Number of changes.
         """
         levenshtein = 0
         insertions = 0
         deletions = 0
         for op, data in diffs:
             if op == self.DIFF_INSERT:
                 insertions += len(data)
             elif op == self.DIFF_DELETE:
                 deletions += len(data)
             elif op == self.DIFF_EQUAL:
                 # A deletion and an insertion is one substitution.
                 levenshtein += max(insertions, deletions)
                 insertions = 0
                 deletions = 0
         levenshtein += max(insertions, deletions)
         return levenshtein
 
     def diff_toDelta(self, diffs):
         """Crush the diff into an encoded string which describes the operations
         required to transform text1 into text2.
         E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
         Operations are tab-separated.  Inserted text is escaped using %xx notation.
 
         Args:
           diffs: Array of diff tuples.
 
         Returns:
           Delta text.
         """
         text = []
         for op, data in diffs:
             if op == self.DIFF_INSERT:
                 # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
                 data = data.encode("utf-8")
                 text.append("+" + urllib.parse.quote(data, "!~*'();/?:@&=+$,# "))
             elif op == self.DIFF_DELETE:
                 text.append("-%d" % len(data))
             elif op == self.DIFF_EQUAL:
                 text.append("=%d" % len(data))
         return "\t".join(text)
 
     def diff_fromDelta(self, text1, delta):
         """Given the original text1, and an encoded string which describes the
         operations required to transform text1 into text2, compute the full diff.
 
         Args:
           text1: Source string for the diff.
           delta: Delta text.
 
         Returns:
           Array of diff tuples.
 
         Raises:
           ValueError: If invalid input.
         """
         if type(delta) == str:
             # Deltas should be composed of a subset of ascii chars, Unicode not
             # required.  If this encode raises UnicodeEncodeError, delta is invalid.
             delta = delta.encode("ascii")
         diffs = []
         pointer = 0  # Cursor in text1
         tokens = delta.split("\t")
         for token in tokens:
             if token == "":
                 # Blank tokens are ok (from a trailing \t).
                 continue
             # Each token begins with a one character parameter which specifies the
             # operation of this token (delete, insert, equality).
             param = token[1:]
             if token[0] == "+":
                 param = urllib.parse.unquote(param)
                 diffs.append((self.DIFF_INSERT, param))
             elif token[0] == "-" or token[0] == "=":
                 try:
                     n = int(param)
                 except ValueError:
                     raise ValueError("Invalid number in diff_fromDelta: " + param)
                 if n < 0:
                     raise ValueError("Negative number in diff_fromDelta: " + param)
                 text = text1[pointer : pointer + n]
                 pointer += n
                 if token[0] == "=":
                     diffs.append((self.DIFF_EQUAL, text))
                 else:
                     diffs.append((self.DIFF_DELETE, text))
             else:
                 # Anything else is an error.
                 raise ValueError(
                     "Invalid diff operation in diff_fromDelta: " + token[0]
                 )
         if pointer != len(text1):
             raise ValueError(
                 "Delta length (%d) does not equal source text length (%d)."
                 % (pointer, len(text1))
             )
         return diffs
 
     #  MATCH FUNCTIONS
 
     def match_main(self, text, pattern, loc):
         """Locate the best instance of 'pattern' in 'text' near 'loc'.
 
         Args:
           text: The text to search.
           pattern: The pattern to search for.
           loc: The location to search around.
 
         Returns:
           Best match index or -1.
         """
         # Check for null inputs.
         if text is None or pattern is None:
             raise ValueError("Null inputs. (match_main)")
 
         loc = max(0, min(loc, len(text)))
         if text == pattern:
             # Shortcut (potentially not guaranteed by the algorithm)
             return 0
         elif not text:
             # Nothing to match.
             return -1
         elif text[loc : loc + len(pattern)] == pattern:
             # Perfect match at the perfect spot!  (Includes case of null pattern)
             return loc
         else:
             # Do a fuzzy compare.
             match = self.match_bitap(text, pattern, loc)
             return match
 
     def match_bitap(self, text, pattern, loc):
         """Locate the best instance of 'pattern' in 'text' near 'loc' using the
         Bitap algorithm.
 
         Args:
           text: The text to search.
           pattern: The pattern to search for.
           loc: The location to search around.
 
         Returns:
           Best match index or -1.
         """
         # Python doesn't have a maxint limit, so ignore this check.
         # if self.Match_MaxBits != 0 and len(pattern) > self.Match_MaxBits:
         #  raise ValueError("Pattern too long for this application.")
 
         # Initialise the alphabet.
         s = self.match_alphabet(pattern)
 
         def match_bitapScore(e, x):
             """Compute and return the score for a match with e errors and x location.
             Accesses loc and pattern through being a closure.
 
             Args:
               e: Number of errors in match.
               x: Location of match.
 
             Returns:
               Overall score for match (0.0 = good, 1.0 = bad).
             """
             accuracy = float(e) / len(pattern)
             proximity = abs(loc - x)
             if not self.Match_Distance:
                 # Dodge divide by zero error.
                 return proximity and 1.0 or accuracy
             return accuracy + (proximity / float(self.Match_Distance))
 
         # Highest score beyond which we give up.
         score_threshold = self.Match_Threshold
         # Is there a nearby exact match? (speedup)
         best_loc = text.find(pattern, loc)
         if best_loc != -1:
             score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
             # What about in the other direction? (speedup)
             best_loc = text.rfind(pattern, loc + len(pattern))
             if best_loc != -1:
                 score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
 
         # Initialise the bit arrays.
         matchmask = 1 << (len(pattern) - 1)
         best_loc = -1
 
         bin_max = len(pattern) + len(text)
         # Empty initialization added to appease pychecker.
         last_rd = None
         for d in range(len(pattern)):
             # Scan for the best match each iteration allows for one more error.
             # Run a binary search to determine how far from 'loc' we can stray at
             # this error level.
             bin_min = 0
             bin_mid = bin_max
             while bin_min < bin_mid:
                 if match_bitapScore(d, loc + bin_mid) <= score_threshold:
                     bin_min = bin_mid
                 else:
                     bin_max = bin_mid
                 bin_mid = (bin_max - bin_min) // 2 + bin_min
 
             # Use the result from this iteration as the maximum for the next.
             bin_max = bin_mid
             start = max(1, loc - bin_mid + 1)
             finish = min(loc + bin_mid, len(text)) + len(pattern)
 
             rd = [0] * (finish + 2)
             rd[finish + 1] = (1 << d) - 1
             for j in range(finish, start - 1, -1):
                 if len(text) <= j - 1:
                     # Out of range.
                     charMatch = 0
                 else:
                     charMatch = s.get(text[j - 1], 0)
                 if d == 0:  # First pass: exact match.
                     rd[j] = ((rd[j + 1] << 1) | 1) & charMatch
                 else:  # Subsequent passes: fuzzy match.
                     rd[j] = (
                         (((rd[j + 1] << 1) | 1) & charMatch)
                         | (((last_rd[j + 1] | last_rd[j]) << 1) | 1)
                         | last_rd[j + 1]
                     )
                 if rd[j] & matchmask:
                     score = match_bitapScore(d, j - 1)
                     # This match will almost certainly be better than any existing match.
                     # But check anyway.
                     if score <= score_threshold:
                         # Told you so.
                         score_threshold = score
                         best_loc = j - 1
                         if best_loc > loc:
                             # When passing loc, don't exceed our current distance from loc.
                             start = max(1, 2 * loc - best_loc)
                         else:
                             # Already passed loc, downhill from here on in.
                             break
             # No hope for a (better) match at greater error levels.
             if match_bitapScore(d + 1, loc) > score_threshold:
                 break
             last_rd = rd
         return best_loc
 
     def match_alphabet(self, pattern):
         """Initialise the alphabet for the Bitap algorithm.
 
         Args:
           pattern: The text to encode.
 
         Returns:
           Hash of character locations.
         """
         s = {}
         for char in pattern:
             s[char] = 0
         for i in range(len(pattern)):
             s[pattern[i]] |= 1 << (len(pattern) - i - 1)
         return s
 
     #  PATCH FUNCTIONS
 
     def patch_addContext(self, patch, text):
         """Increase the context until it is unique,
         but don't let the pattern expand beyond Match_MaxBits.
 
         Args:
           patch: The patch to grow.
           text: Source text.
         """
         if len(text) == 0:
             return
         pattern = text[patch.start2 : patch.start2 + patch.length1]
         padding = 0
 
         # Look for the first and last matches of pattern in text.  If two different
         # matches are found, increase the pattern length.
         while text.find(pattern) != text.rfind(pattern) and (
             self.Match_MaxBits == 0
             or len(pattern) < self.Match_MaxBits - self.Patch_Margin - self.Patch_Margin
         ):
             padding += self.Patch_Margin
             pattern = text[
                 max(0, patch.start2 - padding) : patch.start2 + patch.length1 + padding
             ]
         # Add one chunk for good luck.
         padding += self.Patch_Margin
 
         # Add the prefix.
         prefix = text[max(0, patch.start2 - padding) : patch.start2]
         if prefix:
             patch.diffs[:0] = [(self.DIFF_EQUAL, prefix)]
         # Add the suffix.
         suffix = text[
             patch.start2 + patch.length1 : patch.start2 + patch.length1 + padding
         ]
         if suffix:
             patch.diffs.append((self.DIFF_EQUAL, suffix))
 
         # Roll back the start points.
         patch.start1 -= len(prefix)
         patch.start2 -= len(prefix)
         # Extend lengths.
         patch.length1 += len(prefix) + len(suffix)
         patch.length2 += len(prefix) + len(suffix)
 
     def patch_make(self, a, b=None, c=None):
         """Compute a list of patches to turn text1 into text2.
         Use diffs if provided, otherwise compute it ourselves.
         There are four ways to call this function, depending on what data is
         available to the caller:
         Method 1:
         a = text1, b = text2
         Method 2:
         a = diffs
         Method 3 (optimal):
         a = text1, b = diffs
         Method 4 (deprecated, use method 3):
         a = text1, b = text2, c = diffs
 
         Args:
           a: text1 (methods 1,3,4) or Array of diff tuples for text1 to
               text2 (method 2).
           b: text2 (methods 1,4) or Array of diff tuples for text1 to
               text2 (method 3) or undefined (method 2).
           c: Array of diff tuples for text1 to text2 (method 4) or
               undefined (methods 1,2,3).
 
         Returns:
           Array of Patch objects.
         """
         text1 = None
         diffs = None
         # Note that texts may arrive as 'str' or 'unicode'.
         if isinstance(a, str) and isinstance(b, str) and c is None:
             # Method 1: text1, text2
             # Compute diffs from text1 and text2.
             text1 = a
             diffs = self.diff_main(text1, b, True)
             if len(diffs) > 2:
                 self.diff_cleanupSemantic(diffs)
                 self.diff_cleanupEfficiency(diffs)
         elif isinstance(a, list) and b is None and c is None:
             # Method 2: diffs
             # Compute text1 from diffs.
             diffs = a
             text1 = self.diff_text1(diffs)
         elif isinstance(a, str) and isinstance(b, list) and c is None:
             # Method 3: text1, diffs
             text1 = a
             diffs = b
         elif isinstance(a, str) and isinstance(b, str) and isinstance(c, list):
             # Method 4: text1, text2, diffs
             # text2 is not used.
             text1 = a
             diffs = c
         else:
             raise ValueError("Unknown call format to patch_make.")
 
         if not diffs:
             return []  # Get rid of the None case.
         patches = []
         patch = patch_obj()
         char_count1 = 0  # Number of characters into the text1 string.
         char_count2 = 0  # Number of characters into the text2 string.
         prepatch_text = text1  # Recreate the patches to determine context info.
         postpatch_text = text1
         for x in range(len(diffs)):
             (diff_type, diff_text) = diffs[x]
             if len(patch.diffs) == 0 and diff_type != self.DIFF_EQUAL:
                 # A new patch starts here.
                 patch.start1 = char_count1
                 patch.start2 = char_count2
             if diff_type == self.DIFF_INSERT:
                 # Insertion
                 patch.diffs.append(diffs[x])
                 patch.length2 += len(diff_text)
                 postpatch_text = (
                     postpatch_text[:char_count2]
                     + diff_text
                     + postpatch_text[char_count2:]
                 )
             elif diff_type == self.DIFF_DELETE:
                 # Deletion.
                 patch.length1 += len(diff_text)
                 patch.diffs.append(diffs[x])
                 postpatch_text = (
                     postpatch_text[:char_count2]
                     + postpatch_text[char_count2 + len(diff_text) :]
                 )
             elif (
                 diff_type == self.DIFF_EQUAL
                 and len(diff_text) <= 2 * self.Patch_Margin
                 and len(patch.diffs) != 0
                 and len(diffs) != x + 1
             ):
                 # Small equality inside a patch.
                 patch.diffs.append(diffs[x])
                 patch.length1 += len(diff_text)
                 patch.length2 += len(diff_text)
 
             if diff_type == self.DIFF_EQUAL and len(diff_text) >= 2 * self.Patch_Margin:
                 # Time for a new patch.
                 if len(patch.diffs) != 0:
                     self.patch_addContext(patch, prepatch_text)
                     patches.append(patch)
                     patch = patch_obj()
                     # Unlike Unidiff, our patch lists have a rolling context.
                     # http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
                     # Update prepatch text & pos to reflect the application of the
                     # just completed patch.
                     prepatch_text = postpatch_text
                     char_count1 = char_count2
 
             # Update the current character count.
             if diff_type != self.DIFF_INSERT:
                 char_count1 += len(diff_text)
             if diff_type != self.DIFF_DELETE:
                 char_count2 += len(diff_text)
 
         # Pick up the leftover patch if not empty.
         if len(patch.diffs) != 0:
             self.patch_addContext(patch, prepatch_text)
             patches.append(patch)
         return patches
 
     def patch_deepCopy(self, patches):
         """Given an array of patches, return another array that is identical.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           Array of Patch objects.
         """
         patchesCopy = []
         for patch in patches:
             patchCopy = patch_obj()
             # No need to deep copy the tuples since they are immutable.
             patchCopy.diffs = patch.diffs[:]
             patchCopy.start1 = patch.start1
             patchCopy.start2 = patch.start2
             patchCopy.length1 = patch.length1
             patchCopy.length2 = patch.length2
             patchesCopy.append(patchCopy)
         return patchesCopy
 
     def patch_apply(self, patches, text):
         """Merge a set of patches onto the text.  Return a patched text, as well
         as a list of true/false values indicating which patches were applied.
 
         Args:
           patches: Array of Patch objects.
           text: Old text.
 
         Returns:
           Two element Array, containing the new text and an array of boolean values.
         """
         if not patches:
             return (text, [])
 
         # Deep copy the patches so that no changes are made to originals.
         patches = self.patch_deepCopy(patches)
 
         nullPadding = self.patch_addPadding(patches)
         text = nullPadding + text + nullPadding
         self.patch_splitMax(patches)
 
         # delta keeps track of the offset between the expected and actual location
         # of the previous patch.  If there are patches expected at positions 10 and
         # 20, but the first patch was found at 12, delta is 2 and the second patch
         # has an effective expected position of 22.
         delta = 0
         results = []
         for patch in patches:
             expected_loc = patch.start2 + delta
             text1 = self.diff_text1(patch.diffs)
             end_loc = -1
             if len(text1) > self.Match_MaxBits:
                 # patch_splitMax will only provide an oversized pattern in the case of
                 # a monster delete.
                 start_loc = self.match_main(
                     text, text1[: self.Match_MaxBits], expected_loc
                 )
                 if start_loc != -1:
                     end_loc = self.match_main(
                         text,
                         text1[-self.Match_MaxBits :],
                         expected_loc + len(text1) - self.Match_MaxBits,
                     )
                     if end_loc == -1 or start_loc >= end_loc:
                         # Can't find valid trailing context.  Drop this patch.
                         start_loc = -1
             else:
                 start_loc = self.match_main(text, text1, expected_loc)
             if start_loc == -1:
                 # No match found.  :(
                 results.append(False)
                 # Subtract the delta for this failed patch from subsequent patches.
                 delta -= patch.length2 - patch.length1
             else:
                 # Found a match.  :)
                 results.append(True)
                 delta = start_loc - expected_loc
                 if end_loc == -1:
                     text2 = text[start_loc : start_loc + len(text1)]
                 else:
                     text2 = text[start_loc : end_loc + self.Match_MaxBits]
                 if text1 == text2:
                     # Perfect match, just shove the replacement text in.
                     text = (
                         text[:start_loc]
                         + self.diff_text2(patch.diffs)
                         + text[start_loc + len(text1) :]
                     )
                 else:
                     # Imperfect match.
                     # Run a diff to get a framework of equivalent indices.
                     diffs = self.diff_main(text1, text2, False)
                     if (
                         len(text1) > self.Match_MaxBits
                         and self.diff_levenshtein(diffs) / float(len(text1))
                         > self.Patch_DeleteThreshold
                     ):
                         # The end points match, but the content is unacceptably bad.
                         results[-1] = False
                     else:
                         self.diff_cleanupSemanticLossless(diffs)
                         index1 = 0
                         for op, data in patch.diffs:
                             if op != self.DIFF_EQUAL:
                                 index2 = self.diff_xIndex(diffs, index1)
                             if op == self.DIFF_INSERT:  # Insertion
                                 text = (
                                     text[: start_loc + index2]
                                     + data
                                     + text[start_loc + index2 :]
                                 )
                             elif op == self.DIFF_DELETE:  # Deletion
                                 text = (
                                     text[: start_loc + index2]
                                     + text[
                                         start_loc
                                         + self.diff_xIndex(diffs, index1 + len(data)) :
                                     ]
                                 )
                             if op != self.DIFF_DELETE:
                                 index1 += len(data)
         # Strip the padding off.
         text = text[len(nullPadding) : -len(nullPadding)]
         return (text, results)
 
     def patch_addPadding(self, patches):
         """Add some padding on text start and end so that edges can match
         something.  Intended to be called only from within patch_apply.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           The padding string added to each side.
         """
         paddingLength = self.Patch_Margin
         nullPadding = ""
         for x in range(1, paddingLength + 1):
             nullPadding += chr(x)
 
         # Bump all the patches forward.
         for patch in patches:
             patch.start1 += paddingLength
             patch.start2 += paddingLength
 
         # Add some padding on start of first diff.
         patch = patches[0]
         diffs = patch.diffs
         if not diffs or diffs[0][0] != self.DIFF_EQUAL:
             # Add nullPadding equality.
             diffs.insert(0, (self.DIFF_EQUAL, nullPadding))
             patch.start1 -= paddingLength  # Should be 0.
             patch.start2 -= paddingLength  # Should be 0.
             patch.length1 += paddingLength
             patch.length2 += paddingLength
         elif paddingLength > len(diffs[0][1]):
             # Grow first equality.
             extraLength = paddingLength - len(diffs[0][1])
             newText = nullPadding[len(diffs[0][1]) :] + diffs[0][1]
             diffs[0] = (diffs[0][0], newText)
             patch.start1 -= extraLength
             patch.start2 -= extraLength
             patch.length1 += extraLength
             patch.length2 += extraLength
 
         # Add some padding on end of last diff.
         patch = patches[-1]
         diffs = patch.diffs
         if not diffs or diffs[-1][0] != self.DIFF_EQUAL:
             # Add nullPadding equality.
             diffs.append((self.DIFF_EQUAL, nullPadding))
             patch.length1 += paddingLength
             patch.length2 += paddingLength
         elif paddingLength > len(diffs[-1][1]):
             # Grow last equality.
             extraLength = paddingLength - len(diffs[-1][1])
             newText = diffs[-1][1] + nullPadding[:extraLength]
             diffs[-1] = (diffs[-1][0], newText)
             patch.length1 += extraLength
             patch.length2 += extraLength
 
         return nullPadding
 
     def patch_splitMax(self, patches):
         """Look through the patches and break up any which are longer than the
         maximum limit of the match algorithm.
         Intended to be called only from within patch_apply.
 
         Args:
           patches: Array of Patch objects.
         """
         patch_size = self.Match_MaxBits
         if patch_size == 0:
             # Python has the option of not splitting strings due to its ability
             # to handle integers of arbitrary precision.
             return
         for x in range(len(patches)):
             if patches[x].length1 <= patch_size:
                 continue
             bigpatch = patches[x]
             # Remove the big old patch.
             del patches[x]
             x -= 1
             start1 = bigpatch.start1
             start2 = bigpatch.start2
             precontext = ""
             while len(bigpatch.diffs) != 0:
                 # Create one of several smaller patches.
                 patch = patch_obj()
                 empty = True
                 patch.start1 = start1 - len(precontext)
                 patch.start2 = start2 - len(precontext)
                 if precontext:
                     patch.length1 = patch.length2 = len(precontext)
                     patch.diffs.append((self.DIFF_EQUAL, precontext))
 
                 while (
                     len(bigpatch.diffs) != 0
                     and patch.length1 < patch_size - self.Patch_Margin
                 ):
                     (diff_type, diff_text) = bigpatch.diffs[0]
                     if diff_type == self.DIFF_INSERT:
                         # Insertions are harmless.
                         patch.length2 += len(diff_text)
                         start2 += len(diff_text)
                         patch.diffs.append(bigpatch.diffs.pop(0))
                         empty = False
                     elif (
                         diff_type == self.DIFF_DELETE
                         and len(patch.diffs) == 1
                         and patch.diffs[0][0] == self.DIFF_EQUAL
                         and len(diff_text) > 2 * patch_size
                     ):
                         # This is a large deletion.  Let it pass in one chunk.
                         patch.length1 += len(diff_text)
                         start1 += len(diff_text)
                         empty = False
                         patch.diffs.append((diff_type, diff_text))
                         del bigpatch.diffs[0]
                     else:
                         # Deletion or equality.  Only take as much as we can stomach.
                         diff_text = diff_text[
                             : patch_size - patch.length1 - self.Patch_Margin
                         ]
                         patch.length1 += len(diff_text)
                         start1 += len(diff_text)
                         if diff_type == self.DIFF_EQUAL:
                             patch.length2 += len(diff_text)
                             start2 += len(diff_text)
                         else:
                             empty = False
 
                         patch.diffs.append((diff_type, diff_text))
                         if diff_text == bigpatch.diffs[0][1]:
                             del bigpatch.diffs[0]
                         else:
                             bigpatch.diffs[0] = (
                                 bigpatch.diffs[0][0],
                                 bigpatch.diffs[0][1][len(diff_text) :],
                             )
 
                 # Compute the head context for the next patch.
                 precontext = self.diff_text2(patch.diffs)
                 precontext = precontext[-self.Patch_Margin :]
                 # Append the end context for this patch.
                 postcontext = self.diff_text1(bigpatch.diffs)[: self.Patch_Margin]
                 if postcontext:
                     patch.length1 += len(postcontext)
                     patch.length2 += len(postcontext)
                     if len(patch.diffs) != 0 and patch.diffs[-1][0] == self.DIFF_EQUAL:
                         patch.diffs[-1] = (
                             self.DIFF_EQUAL,
                             patch.diffs[-1][1] + postcontext,
                         )
                     else:
                         patch.diffs.append((self.DIFF_EQUAL, postcontext))
 
                 if not empty:
                     x += 1
                     patches.insert(x, patch)
 
     def patch_toText(self, patches):
         """Take a list of patches and return a textual representation.
 
         Args:
           patches: Array of Patch objects.
 
         Returns:
           Text representation of patches.
         """
         text = []
         for patch in patches:
             text.append(str(patch))
         return "".join(text)
 
     def patch_fromText(self, textline):
         """Parse a textual representation of patches and return a list of patch
         objects.
 
         Args:
           textline: Text representation of patches.
 
         Returns:
           Array of Patch objects.
 
         Raises:
           ValueError: If invalid input.
         """
-        if type(textline) == unicode:
+        if type(textline) == str:
             # Patches should be composed of a subset of ascii chars, Unicode not
             # required.  If this encode raises UnicodeEncodeError, patch is invalid.
             textline = textline.encode("ascii")
         patches = []
         if not textline:
             return patches
         text = textline.split("\n")
         while len(text) != 0:
             m = re.match("^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$", text[0])
             if not m:
                 raise ValueError("Invalid patch string: " + text[0])
             patch = patch_obj()
             patches.append(patch)
             patch.start1 = int(m.group(1))
             if m.group(2) == "":
                 patch.start1 -= 1
                 patch.length1 = 1
             elif m.group(2) == "0":
                 patch.length1 = 0
             else:
                 patch.start1 -= 1
                 patch.length1 = int(m.group(2))
 
             patch.start2 = int(m.group(3))
             if m.group(4) == "":
                 patch.start2 -= 1
                 patch.length2 = 1
             elif m.group(4) == "0":
                 patch.length2 = 0
             else:
                 patch.start2 -= 1
                 patch.length2 = int(m.group(4))
 
             del text[0]
 
             while len(text) != 0:
                 if text[0]:
                     sign = text[0][0]
                 else:
                     sign = ""
                 line = urllib.parse.unquote(text[0][1:])
                 line = line.decode("utf-8")
                 if sign == "+":
                     # Insertion.
                     patch.diffs.append((self.DIFF_INSERT, line))
                 elif sign == "-":
                     # Deletion.
                     patch.diffs.append((self.DIFF_DELETE, line))
                 elif sign == " ":
                     # Minor equality.
                     patch.diffs.append((self.DIFF_EQUAL, line))
                 elif sign == "@":
                     # Start of next patch.
                     break
                 elif sign == "":
                     # Blank line?  Whatever.
                     pass
                 else:
                     # WTF?
                     raise ValueError("Invalid patch mode: '%s'\n%s" % (sign, line))
                 del text[0]
         return patches
 
 
 class patch_obj:
     """Class representing one patch operation."""
 
     def __init__(self):
         """Initializes with an empty list of diffs."""
         self.diffs = []
         self.start1 = None
         self.start2 = None
         self.length1 = 0
         self.length2 = 0
 
     def __str__(self):
         """Emmulate GNU diff's format.
         Header: @@ -382,8 +481,9 @@
         Indicies are printed as 1-based, not 0-based.
 
         Returns:
           The GNU diff string.
         """
         if self.length1 == 0:
             coords1 = str(self.start1) + ",0"
         elif self.length1 == 1:
             coords1 = str(self.start1 + 1)
         else:
             coords1 = str(self.start1 + 1) + "," + str(self.length1)
         if self.length2 == 0:
             coords2 = str(self.start2) + ",0"
         elif self.length2 == 1:
             coords2 = str(self.start2 + 1)
         else:
             coords2 = str(self.start2 + 1) + "," + str(self.length2)
         text = ["@@ -", coords1, " +", coords2, " @@\n"]
         # Escape the body of the patch with %xx notation.
         for op, data in self.diffs:
             if op == diff_match_patch.DIFF_INSERT:
                 text.append("+")
             elif op == diff_match_patch.DIFF_DELETE:
                 text.append("-")
             elif op == diff_match_patch.DIFF_EQUAL:
                 text.append(" ")
             # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
             data = data.encode("utf-8")
             text.append(urllib.parse.quote(data, "!~*'();/?:@&=+$,# ") + "\n")
         return "".join(text)

rhodecode/lib/diffs.py

@@ -1,1272 +1,1271 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of diffing helpers, previously part of vcs
 """
 
 import os
 import re
 import bz2
 import gzip
 import time
 
 import collections
 import difflib
 import logging
 import pickle
 from itertools import tee
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
 # define max context, a file with more than this numbers of lines is unusable
 # in browser anyway
 MAX_CONTEXT = 20 * 1024
 DEFAULT_CONTEXT = 3
 
 
 def get_diff_context(request):
     return MAX_CONTEXT if request.GET.get('fullcontext', '') == '1' else DEFAULT_CONTEXT
 
 
 def get_diff_whitespace_flag(request):
     return request.GET.get('ignorews', '') == '1'
 
 
 class OPS(object):
     ADD = 'A'
     MOD = 'M'
     DEL = 'D'
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     # protect against IntOverflow when passing HUGE context
     if context > MAX_CONTEXT:
         context = MAX_CONTEXT
 
-    submodules = filter(lambda o: isinstance(o, SubModuleNode),
-                        [filenode_new, filenode_old])
+    submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError(
                 "Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.commit.repository
     old_commit = filenode_old.commit or repo.EMPTY_COMMIT
     new_commit = filenode_new.commit
 
     vcs_gitdiff = repo.get_diff(
         old_commit, new_commit, filenode_new.path,
         ignore_whitespace, context, path1=filenode_old.path)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __getitem__(self, key):
         return self.diff.__getitem__(key)
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class Action(object):
     """
     Contains constants for the action value of the lines in a parsed diff.
     """
 
     ADD = 'add'
     DELETE = 'del'
     UNMODIFIED = 'unmod'
 
     CONTEXT = 'context'
     OLD_NO_NL = 'old-no-nl'
     NEW_NO_NL = 'new-no-nl'
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
 
     .. note:: Unicode handling
 
        The original diffs are a byte sequence and can contain filenames
        in mixed encodings. This class generally returns `unicode` objects
        since the result is intended for presentation to the user.
 
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
 
     # used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     # collapse ranges of commits over given number
     _collapse_commits_over = 5
 
     def __init__(self, diff, format='gitdiff', diff_limit=None,
                  file_limit=None, show_full_diff=True):
         """
         :param diff: A `Diff` object representing a diff from a vcs backend
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_limit = diff_limit
         self.file_limit = file_limit
         self.show_full_diff = show_full_diff
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
 
         log.debug('Initialized DiffProcessor with %s mode', format)
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._new_parse_gitdiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         """
         self.cur_diff_size += len(string)
 
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return string \
             .replace('&', '&amp;')\
             .replace('<', '&lt;')\
             .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == Action.DELETE:
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == Action.ADD:
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             # only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
 
             diff = map(self._escaper, self.diff_splitter(chunk.diff))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be copy, rename or chmod
 
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     raw_diff, chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
                     exceeds_limit = len(raw_diff) > self.file_limit
 
                     # changed from _escaper function so we validate size of
                     # each file instead of the whole diff
                     # diff will hide big files but still show small ones
                     # from my tests, big files are fairly safe to be parsed
                     # but the browser is the bottleneck
                     if not self.show_full_diff and exceeds_limit:
                         raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(0, [{
                                   'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].items()
                               if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = next(lineiter)
                         if line['action'] not in (
                                 Action.UNMODIFIED, Action.CONTEXT):
                             nextline = next(lineiter)
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _check_large_diff(self):
         if self.diff_limit:
             log.debug('Checking if diff exceeds current diff_limit of %s', self.diff_limit)
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit `%s` Exceeded', self.diff_limit)
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_gitdiff
     def _new_parse_gitdiff(self, inline_diff=True):
         _files = []
 
         # this can be overriden later to a LimitedDiffContainer type
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
             log.debug('parsing diff %r', head)
 
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'old_mode': None,
                 'new_mode': None,
                 'ops': {},
             }
             if head['old_mode']:
                 stats['old_mode'] = head['old_mode']
             if head['new_mode']:
                 stats['new_mode'] = head['new_mode']
             if head['b_mode']:
                 stats['new_mode'] = head['b_mode']
 
             # delete file
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             # new file
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['old_mode'] = None
                 stats['new_mode'] = head['new_file_mode']
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
 
             # modify operation, can be copy, rename or chmod
             else:
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
 
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['renamed'] = (head['rename_from'], head['rename_to'])
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['copied'] = (head['copy_from'], head['copy_to'])
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['new_file'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 # simulate splitlines, so we keep the line end part
                 diff = self.diff_splitter(chunk.diff)
 
                 # append each file to the diff size
                 raw_chunk_size = len(raw_diff)
 
                 exceeds_limit = raw_chunk_size > self.file_limit
                 self.cur_diff_size += raw_chunk_size
 
                 try:
                     # Check each file instead of the whole diff.
                     # Diff will hide big files but still show small ones.
                     # From the tests big files are fairly safe to be parsed
                     # but the browser is the bottleneck.
                     if not self.show_full_diff and exceeds_limit:
                         log.debug('File `%s` exceeds current file_limit of %s',
                                   safe_unicode(head['b_path']), self.file_limit)
                         raise DiffLimitExceeded(
                             'File Limit %s Exceeded', self.file_limit)
 
                     self._check_large_diff()
 
                     raw_diff, chunks, _stats = self._new_parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             # Hide content of deleted node by setting empty chunks
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(
                 0, [{'old_lineno': '',
                      'new_lineno': '',
                      'action': Action.CONTEXT,
                      'line': msg,
                      } for _op, msg in stats['ops'].items()
                     if _op not in [MOD_FILENODE]])
 
             original_filename = safe_unicode(head['a_path'])
             _files.append({
                 'original_filename': original_filename,
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         return diff_container(sorted(_files, key=sorter))
 
     # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
     def _parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     Action.CONTEXT,
                             'line':       line,
                         })
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       Action.CONTEXT,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return ''.join(raw_diff), chunks, stats
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_lines
     def _new_parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 # match header e.g @@ -0,0 +1 @@\n'
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
 
                 lines = []
                 hunk = {
                     'section_header': gr[-1],
                     'source_start': old_line,
                     'source_length': old_end,
                     'target_start': new_line,
                     'target_length': new_end,
                     'lines': lines,
                 }
                 chunks.append(hunk)
 
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                     raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         if affects_old:
                             action = Action.OLD_NO_NL
                         elif affects_new:
                             action = Action.NEW_NO_NL
                         else:
                             raise Exception('invalid context for no newline')
 
                         lines.append({
                             'old_lineno':   None,
                             'new_lineno':   None,
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
 
         return ''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     @classmethod
     def diff_splitter(cls, string):
         """
         Diff split that emulates .splitlines() but works only on \n
         """
         if not string:
             return
         elif string == '\n':
-            yield u'\n'
+            yield '\n'
         else:
 
             has_newline = string.endswith('\n')
             elements = string.split('\n')
             if has_newline:
                 # skip last element as it's empty string from newlines
                 elements = elements[:-1]
 
             len_elements = len(elements)
 
             for cnt, line in enumerate(elements, start=1):
                 last_line = cnt == len_elements
                 if last_line and not has_newline:
                     yield safe_unicode(line)
                 else:
                     yield safe_unicode(line) + '\n'
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering.
 
         :return: A list of dicts with diff information.
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw diff as a byte string
         """
         return self._diff.raw
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         # TODO(marcink): not sure how to pass in translator
         # here in an efficient way, leave the _ for proper gettext extraction
         _ = lambda s: s
 
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s" class="tooltip"
                 title="%(title)s">%(label)s</a>''' % {
                     'title': _('Click to select line'),
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
 
                     if change['action'] != Action.CONTEXT:
                         anchor_link = True
                     else:
                         anchor_link = False
 
                     ###########################################################
                     # COMMENT ICONS
                     ###########################################################
                     _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
 
                     if enable_comments and change['action'] != Action.CONTEXT:
                         _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
 
                     _html.append('''</span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>\n''')
 
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     code_classes = [code_class]
                     if (not enable_comments or
                             change['action'] == Action.CONTEXT):
                         code_classes.append('no-comment')
                     _html.append('\t<td class="%s">' % ' '.join(code_classes))
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
     def get_context_of_line(
             self, path, diff_line=None, context_before=3, context_after=3):
         """
         Returns the context lines for the specified diff line.
 
         :type diff_line: :class:`DiffLineNumber`
         """
         assert self.parsed, "DiffProcessor is not initialized."
 
         if None not in diff_line:
             raise ValueError(
                 "Cannot specify both line numbers: {}".format(diff_line))
 
         file_diff = self._get_file_diff(path)
         chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
 
         first_line_to_include = max(idx - context_before, 0)
         first_line_after_context = idx + context_after + 1
         context_lines = chunk[first_line_to_include:first_line_after_context]
 
         line_contents = [
             _context_line(line) for line in context_lines
             if _is_diff_content(line)]
         # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
         # Once they are fixed, we can drop this line here.
         if line_contents:
             line_contents[-1] = (
                 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
         return line_contents
 
     def find_context(self, path, context, offset=0):
         """
         Finds the given `context` inside of the diff.
 
         Use the parameter `offset` to specify which offset the target line has
         inside of the given `context`. This way the correct diff line will be
         returned.
 
         :param offset: Shall be used to specify the offset of the main line
             within the given `context`.
         """
         if offset < 0 or offset >= len(context):
             raise ValueError(
                 "Only positive values up to the length of the context "
                 "minus one are allowed.")
 
         matches = []
         file_diff = self._get_file_diff(path)
 
         for chunk in file_diff['chunks']:
             context_iter = iter(context)
             for line_idx, line in enumerate(chunk):
                 try:
                     if _context_line(line) == next(context_iter):
                         continue
                 except StopIteration:
                     matches.append((line_idx, chunk))
                 context_iter = iter(context)
 
         # Increment position and triger StopIteration
         # if we had a match at the end
         line_idx += 1
         try:
             next(context_iter)
         except StopIteration:
             matches.append((line_idx, chunk))
 
         effective_offset = len(context) - offset
         found_at_diff_lines = [
             _line_to_diff_line_number(chunk[idx - effective_offset])
             for idx, chunk in matches]
 
         return found_at_diff_lines
 
     def _get_file_diff(self, path):
         for file_diff in self.parsed_diff:
             if file_diff['filename'] == path:
                 break
         else:
             raise FileNotInDiffException("File {} not in diff".format(path))
         return file_diff
 
     def _find_chunk_line_index(self, file_diff, diff_line):
         for chunk in file_diff['chunks']:
             for idx, line in enumerate(chunk):
                 if line['old_lineno'] == diff_line.old:
                     return chunk, idx
                 if line['new_lineno'] == diff_line.new:
                     return chunk, idx
         raise LineNotInDiffException(
             "The line {} is not part of the diff.".format(diff_line))
 
 
 def _is_diff_content(line):
     return line['action'] in (
         Action.UNMODIFIED, Action.ADD, Action.DELETE)
 
 
 def _context_line(line):
     return (line['action'], line['line'])
 
 
 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
 
 
 def _line_to_diff_line_number(line):
     new_line_no = line['new_lineno'] or None
     old_line_no = line['old_lineno'] or None
     return DiffLineNumber(old=old_line_no, new=new_line_no)
 
 
 class FileNotInDiffException(Exception):
     """
     Raised when the context for a missing file is requested.
 
     If you request the context for a line in a file which is not part of the
     given diff, then this exception is raised.
     """
 
 
 class LineNotInDiffException(Exception):
     """
     Raised when the context for a missing line is requested.
 
     If you request the context for a line in a file and this line is not
     part of the given diff, then this exception is raised.
     """
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 # NOTE(marcink): if diffs.mako change, probably this
 # needs a bump to next version
 CURRENT_DIFF_VERSION = 'v5'
 
 
 def _cleanup_cache_file(cached_diff_file):
     # cleanup file to not store it "damaged"
     try:
         os.remove(cached_diff_file)
     except Exception:
         log.exception('Failed to cleanup path %s', cached_diff_file)
 
 
 def _get_compression_mode(cached_diff_file):
     mode = 'bz2'
     if 'mode:plain' in cached_diff_file:
         mode = 'plain'
     elif 'mode:gzip' in cached_diff_file:
         mode = 'gzip'
     return mode
 
 
 def cache_diff(cached_diff_file, diff, commits):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': diff,
         'commits': commits
     }
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         else:
             with bz2.BZ2File(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
     except Exception:
         log.warn('Failed to save cache', exc_info=True)
         _cleanup_cache_file(cached_diff_file)
 
     log.debug('Saved diff cache under %s in %.4fs', cached_diff_file, time.time() - start)
 
 
 def load_cached_diff(cached_diff_file):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     default_struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': None,
         'commits': None
     }
 
     has_cache = os.path.isfile(cached_diff_file)
     if not has_cache:
         log.debug('Reading diff cache file failed %s', cached_diff_file)
         return default_struct
 
     data = None
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         else:
             with bz2.BZ2File(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
     except Exception:
         log.warn('Failed to read diff cache file', exc_info=True)
 
     if not data:
         data = default_struct
 
     if not isinstance(data, dict):
         # old version of data ?
         data = default_struct
 
     # check version
     if data.get('version') != CURRENT_DIFF_VERSION:
         # purge cache
         _cleanup_cache_file(cached_diff_file)
         return default_struct
 
     log.debug('Loaded diff cache from %s in %.4fs', cached_diff_file, time.time() - start)
 
     return data
 
 
 def generate_diff_cache_key(*args):
     """
     Helper to generate a cache key using arguments
     """
     def arg_mapper(input_param):
         input_param = safe_str(input_param)
         # we cannot allow '/' in arguments since it would allow
         # subdirectory usage
         input_param.replace('/', '_')
         return input_param or None  # prevent empty string arguments
 
     return '_'.join([
         '{}' for i in range(len(args))]).format(*map(arg_mapper, args))
 
 
 def diff_cache_exist(cache_storage, *args):
     """
     Based on all generated arguments check and return a cache path
     """
     args = list(args) + ['mode:gzip']
     cache_key = generate_diff_cache_key(*args)
     cache_file_path = os.path.join(cache_storage, cache_key)
     # prevent path traversal attacks using some param that have e.g '../../'
     if not os.path.abspath(cache_file_path).startswith(cache_storage):
         raise ValueError('Final path must be within {}'.format(cache_storage))
 
     return cache_file_path

rhodecode/lib/feedgenerator/feedgenerator.py

@@ -1,444 +1,444 b''
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
 #     1. Redistributions of source code must retain the above copyright notice,
 #        this list of conditions and the following disclaimer.
 #
 #     2. Redistributions in binary form must reproduce the above copyright
 #        notice, this list of conditions and the following disclaimer in the
 #        documentation and/or other materials provided with the distribution.
 #
 #     3. Neither the name of Django nor the names of its contributors may be used
 #        to endorse or promote products derived from this software without
 #        specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """
 For definitions of the different versions of RSS, see:
 http://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004/02/04/incompatible-rss
 """
 
 
 import datetime
-from io import StringIO
+import io
 
 import pytz
 from six.moves.urllib import parse as urlparse
 
 from rhodecode.lib.feedgenerator import datetime_safe
 from rhodecode.lib.feedgenerator.utils import SimplerXMLGenerator, iri_to_uri, force_text
 
 
 #### The following code comes from ``django.utils.feedgenerator`` ####
 
 
 def rfc2822_date(date):
     # We can't use strftime() because it produces locale-dependent results, so
     # we have to map english month and day names manually
     months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',)
     days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     # We do this ourselves to be timezone aware, email.Utils is not tz aware.
     dow = days[date.weekday()]
     month = months[date.month - 1]
     time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month))
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + '-0000'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d%02d' % (hour, minute)
 
 
 def rfc3339_date(date):
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     time_str = date.strftime('%Y-%m-%dT%H:%M:%S')
 
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + 'Z'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d:%02d' % (hour, minute)
 
 
 def get_tag_uri(url, date):
     """
     Creates a TagURI.
 
     See http://web.archive.org/web/20110514113830/http://diveintomark.org/archives/2004/05/28/howto-atom-id
     """
     bits = urlparse(url)
     d = ''
     if date is not None:
         d = ',%s' % datetime_safe.new_datetime(date).strftime('%Y-%m-%d')
     return 'tag:%s%s:%s/%s' % (bits.hostname, d, bits.path, bits.fragment)
 
 
 class SyndicationFeed(object):
     """Base class for all syndication feeds. Subclasses should provide write()"""
 
     def __init__(self, title, link, description, language=None, author_email=None,
                  author_name=None, author_link=None, subtitle=None, categories=None,
                  feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs):
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [force_text(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         self.feed = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'language': to_unicode(language),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'subtitle': to_unicode(subtitle),
             'categories': categories or (),
             'feed_url': iri_to_uri(feed_url),
             'feed_copyright': to_unicode(feed_copyright),
             'id': feed_guid or link,
             'ttl': ttl,
         }
         self.feed.update(kwargs)
         self.items = []
 
     def add_item(self, title, link, description, author_email=None,
                  author_name=None, author_link=None, pubdate=None, comments=None,
                  unique_id=None, unique_id_is_permalink=None, enclosure=None,
                  categories=(), item_copyright=None, ttl=None, updateddate=None,
                  enclosures=None, **kwargs):
         """
         Adds an item to the feed. All args are expected to be Python Unicode
         objects except pubdate and updateddate, which are datetime.datetime
         objects, and enclosures, which is an iterable of instances of the
         Enclosure class.
         """
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [to_unicode(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         if enclosure is None:
             enclosures = [] if enclosures is None else enclosures
 
         item = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'pubdate': pubdate,
             'updateddate': updateddate,
             'comments': to_unicode(comments),
             'unique_id': to_unicode(unique_id),
             'unique_id_is_permalink': unique_id_is_permalink,
             'enclosures': enclosures,
             'categories': categories or (),
             'item_copyright': to_unicode(item_copyright),
             'ttl': ttl,
         }
         item.update(kwargs)
         self.items.append(item)
 
     def num_items(self):
         return len(self.items)
 
     def root_attributes(self):
         """
         Return extra attributes to place on the root (i.e. feed/channel) element.
         Called from write().
         """
         return {}
 
     def add_root_elements(self, handler):
         """
         Add elements in the root (i.e. feed/channel) element. Called
         from write().
         """
         pass
 
     def item_attributes(self, item):
         """
         Return extra attributes to place on each item (i.e. item/entry) element.
         """
         return {}
 
     def add_item_elements(self, handler, item):
         """
         Add elements on each item (i.e. item/entry) element.
         """
         pass
 
     def write(self, outfile, encoding):
         """
         Outputs the feed in the given encoding to outfile, which is a file-like
         object. Subclasses should override this.
         """
         raise NotImplementedError('subclasses of SyndicationFeed must provide a write() method')
 
     def writeString(self, encoding):
         """
         Returns the feed in the given encoding as a string.
         """
-        s = StringIO()
+        s = io.StringIO()
         self.write(s, encoding)
         return s.getvalue()
 
     def latest_post_date(self):
         """
         Returns the latest item's pubdate or updateddate. If no items
         have either of these attributes this returns the current UTC date/time.
         """
         latest_date = None
         date_keys = ('updateddate', 'pubdate')
 
         for item in self.items:
             for date_key in date_keys:
                 item_date = item.get(date_key)
                 if item_date:
                     if latest_date is None or item_date > latest_date:
                         latest_date = item_date
 
         # datetime.now(tz=utc) is slower, as documented in django.utils.timezone.now
         return latest_date or datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
 
 
 class Enclosure(object):
     """Represents an RSS enclosure"""
     def __init__(self, url, length, mime_type):
         """All args are expected to be Python Unicode objects"""
         self.length, self.mime_type = length, mime_type
         self.url = iri_to_uri(url)
 
 
 class RssFeed(SyndicationFeed):
     content_type = 'application/rss+xml; charset=utf-8'
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement("rss", self.rss_attributes())
         handler.startElement("channel", self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         self.endChannelElement(handler)
         handler.endElement("rss")
 
     def rss_attributes(self):
         return {"version": self._version,
                 "xmlns:atom": "http://www.w3.org/2005/Atom"}
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement('item', self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("item")
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", self.feed['link'])
         handler.addQuickElement("description", self.feed['description'])
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("atom:link", None, {"rel": "self", "href": self.feed['feed_url']})
         if self.feed['language'] is not None:
             handler.addQuickElement("language", self.feed['language'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", cat)
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("copyright", self.feed['feed_copyright'])
         handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()))
         if self.feed['ttl'] is not None:
             handler.addQuickElement("ttl", self.feed['ttl'])
 
     def endChannelElement(self, handler):
         handler.endElement("channel")
 
 
 class RssUserland091Feed(RssFeed):
     _version = "0.91"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
 
 class Rss201rev2Feed(RssFeed):
     # Spec: http://blogs.law.harvard.edu/tech/rss
     _version = "2.0"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
         # Author information.
         if item["author_name"] and item["author_email"]:
             handler.addQuickElement("author", "%s (%s)" % (item['author_email'], item['author_name']))
         elif item["author_email"]:
             handler.addQuickElement("author", item["author_email"])
         elif item["author_name"]:
             handler.addQuickElement(
                 "dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}
             )
 
         if item['pubdate'] is not None:
             handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']))
         if item['comments'] is not None:
             handler.addQuickElement("comments", item['comments'])
         if item['unique_id'] is not None:
             guid_attrs = {}
             if isinstance(item.get('unique_id_is_permalink'), bool):
                 guid_attrs['isPermaLink'] = str(item['unique_id_is_permalink']).lower()
             handler.addQuickElement("guid", item['unique_id'], guid_attrs)
         if item['ttl'] is not None:
             handler.addQuickElement("ttl", item['ttl'])
 
         # Enclosure.
         if item['enclosures']:
             enclosures = list(item['enclosures'])
             if len(enclosures) > 1:
                 raise ValueError(
                     "RSS feed items may only have one enclosure, see "
                     "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
                 )
             enclosure = enclosures[0]
             handler.addQuickElement('enclosure', '', {
                 'url': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", cat)
 
 
 class Atom1Feed(SyndicationFeed):
     # Spec: https://tools.ietf.org/html/rfc4287
     content_type = 'application/atom+xml; charset=utf-8'
     ns = "http://www.w3.org/2005/Atom"
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement('feed', self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         handler.endElement("feed")
 
     def root_attributes(self):
         if self.feed['language'] is not None:
             return {"xmlns": self.ns, "xml:lang": self.feed['language']}
         else:
             return {"xmlns": self.ns}
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", "", {"rel": "alternate", "href": self.feed['link']})
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']})
         handler.addQuickElement("id", self.feed['id'])
         handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()))
         if self.feed['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", self.feed['author_name'])
             if self.feed['author_email'] is not None:
                 handler.addQuickElement("email", self.feed['author_email'])
             if self.feed['author_link'] is not None:
                 handler.addQuickElement("uri", self.feed['author_link'])
             handler.endElement("author")
         if self.feed['subtitle'] is not None:
             handler.addQuickElement("subtitle", self.feed['subtitle'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", "", {"term": cat})
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("rights", self.feed['feed_copyright'])
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement("entry", self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("entry")
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"})
 
         if item['pubdate'] is not None:
             handler.addQuickElement('published', rfc3339_date(item['pubdate']))
 
         if item['updateddate'] is not None:
             handler.addQuickElement('updated', rfc3339_date(item['updateddate']))
 
         # Author information.
         if item['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", item['author_name'])
             if item['author_email'] is not None:
                 handler.addQuickElement("email", item['author_email'])
             if item['author_link'] is not None:
                 handler.addQuickElement("uri", item['author_link'])
             handler.endElement("author")
 
         # Unique ID.
         if item['unique_id'] is not None:
             unique_id = item['unique_id']
         else:
             unique_id = get_tag_uri(item['link'], item['pubdate'])
         handler.addQuickElement("id", unique_id)
 
         # Summary.
         if item['description'] is not None:
             handler.addQuickElement("summary", item['description'], {"type": "html"})
 
         # Enclosures.
         for enclosure in item['enclosures']:
             handler.addQuickElement('link', '', {
                 'rel': 'enclosure',
                 'href': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", "", {"term": cat})
 
         # Rights.
         if item['item_copyright'] is not None:
             handler.addQuickElement("rights", item['item_copyright'])
 
 
 # This isolates the decision of what the system default is, so calling code can
 # do "feedgenerator.DefaultFeed" instead of "feedgenerator.Rss201rev2Feed".
 DefaultFeed = Rss201rev2Feed
  No newline at end of file

rhodecode/lib/hooks_base.py

@@ -1,538 +1,538 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
-    vcs_part = safe_str(u'.%s' % repo.repo_type)
+    vcs_part = safe_str('.%s' % repo.repo_type)
     size_vcs, size_root, size_total = _get_scm_size(vcs_part,
                                                     repo.repo_full_path)
     msg = ('Repository `%s` size summary %s:%s repo:%s total:%s\n'
            % (repo.repo_name, vcs_part, size_vcs, size_root, size_total))
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
         if extras.commit_ids and extras.check_branch_perms:
 
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = 'Branch `{}` changes rejected by rule {}. ' \
                                    'FORCE PUSH FORBIDDEN.'.format(branch_name, rule)
                 else:
                     halt_message = 'Branch `{}` changes rejected by rule {}.'.format(
                         branch_name, rule)
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_pull_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `%s`' % (extras.repository,)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_push_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
 
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = 'Released lock on repo `{}`\n'.format(safe_str(extras.repository))
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='branch', ref_name=safe_str(branch_name)))
 
         for book_name in extras.new_refs['bookmarks']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name)))
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = (
         'Repository `%s` locked by user `%s`. Reason:`%s`'
         % (repo_name, user_name, reason))
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/logging_formatter.py

@@ -1,187 +1,187 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import sys
 import logging
 
 
-BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
 
 # Sequences
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[0;%dm"
 BOLD_SEQ = "\033[1m"
 
 COLORS = {
     'CRITICAL': MAGENTA,
     'ERROR': RED,
     'WARNING': CYAN,
     'INFO': GREEN,
     'DEBUG': BLUE,
     'SQL': YELLOW
 }
 
 
 def _inject_req_id(record, with_prefix=True):
     from pyramid.threadlocal import get_current_request
     dummy = '00000000-0000-0000-0000-000000000000'
     req_id = None
 
     req = get_current_request()
     if req:
         req_id = getattr(req, 'req_id', None)
     if with_prefix:
         req_id = 'req_id:%-36s' % (req_id or dummy)
     else:
         req_id = (req_id or dummy)
     record.req_id = req_id
 
 
 def _add_log_to_debug_bucket(formatted_record):
     from pyramid.threadlocal import get_current_request
     req = get_current_request()
     if req:
         req.req_id_bucket.append(formatted_record)
 
 
 def one_space_trim(s):
     if s.find("  ") == -1:
         return s
     else:
         s = s.replace('  ', ' ')
         return one_space_trim(s)
 
 
 def format_sql(sql):
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
         .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
         .replace('FROM', '\n\tFROM')\
         .replace('ORDER BY', '\n\tORDER BY')\
         .replace('LIMIT', '\n\tLIMIT')\
         .replace('WHERE', '\n\tWHERE')\
         .replace('AND', '\n\tAND')\
         .replace('LEFT', '\n\tLEFT')\
         .replace('INNER', '\n\tINNER')\
         .replace('INSERT', '\n\tINSERT')\
         .replace('DELETE', '\n\tDELETE')
     return sql
 
 
 class ExceptionAwareFormatter(logging.Formatter):
     """
     Extended logging formatter which prints out remote tracebacks.
     """
 
     def formatException(self, ei):
         ex_type, ex_value, ex_tb = ei
 
         local_tb = logging.Formatter.formatException(self, ei)
         if hasattr(ex_value, '_vcs_server_traceback'):
 
             def formatRemoteTraceback(remote_tb_lines):
                 result = ["\n +--- This exception occured remotely on VCSServer - Remote traceback:\n\n"]
                 result.append(remote_tb_lines)
                 result.append("\n +--- End of remote traceback\n")
                 return result
 
             try:
                 if ex_type is not None and ex_value is None and ex_tb is None:
                     # possible old (3.x) call syntax where caller is only
                     # providing exception object
                     if type(ex_type) is not type:
                         raise TypeError(
                             "invalid argument: ex_type should be an exception "
                             "type, or just supply no arguments at all")
                 if ex_type is None and ex_tb is None:
                     ex_type, ex_value, ex_tb = sys.exc_info()
 
                 remote_tb = getattr(ex_value, "_vcs_server_traceback", None)
 
                 if remote_tb:
                     remote_tb = formatRemoteTraceback(remote_tb)
                     return local_tb + ''.join(remote_tb)
             finally:
                 # clean up cycle to traceback, to allow proper GC
                 del ex_type, ex_value, ex_tb
 
         return local_tb
 
 
 class RequestTrackingFormatter(ExceptionAwareFormatter):
     def format(self, record):
         _inject_req_id(record)
         def_record = logging.Formatter.format(self, record)
         _add_log_to_debug_bucket(def_record)
         return def_record
 
 
 class ColorFormatter(ExceptionAwareFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorRequestTrackingFormatter(RequestTrackingFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorRequestTrackingFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorFormatterSql(logging.Formatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
 
         start = COLOR_SEQ % (COLORS['SQL'])
         def_record = format_sql(logging.Formatter.format(self, record))
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 # marcink: needs to stay with this name for backward .ini compatability
 Pyro4AwareFormatter = ExceptionAwareFormatter

rhodecode/lib/markup_renderer.py

@@ -1,580 +1,580 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Renderer for markup languages with ability to parse using rst or markdown
 """
 
 import re
 import os
 import lxml
 import logging
 import urllib.parse
 import bleach
 
 from mako.lookup import TemplateLookup
 from mako.template import Template as MakoTemplate
 
 from docutils.core import publish_parts
 from docutils.parsers.rst import directives
 from docutils import writers
 from docutils.writers import html4css1
 import markdown
 
 from rhodecode.lib.markdown_ext import GithubFlavoredMarkdownExtension
 from rhodecode.lib.utils2 import (safe_unicode, md5_safe, MENTIONS_REGEX)
 
 log = logging.getLogger(__name__)
 
 # default renderer used to generate automated comments
 DEFAULT_COMMENTS_RENDERER = 'rst'
 
 try:
     from lxml.html import fromstring
     from lxml.html import tostring
 except ImportError:
     log.exception('Failed to import lxml')
     fromstring = None
     tostring = None
 
 
 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
     """
     Custom HTML Translator used for sandboxing potential
     JS injections in ref links
     """
     def visit_literal_block(self, node):
         self.body.append(self.starttag(node, 'pre', CLASS='codehilite literal-block'))
 
     def visit_reference(self, node):
         if 'refuri' in node.attributes:
             refuri = node['refuri']
             if ':' in refuri:
                 prefix, link = refuri.lstrip().split(':', 1)
                 prefix = prefix or ''
 
                 if prefix.lower() == 'javascript':
                     # we don't allow javascript type of refs...
                     node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
 
         # old style class requires this...
         return html4css1.HTMLTranslator.visit_reference(self, node)
 
 
 class RhodeCodeWriter(writers.html4css1.Writer):
     def __init__(self):
         writers.Writer.__init__(self)
         self.translator_class = CustomHTMLTranslator
 
 
 def relative_links(html_source, server_paths):
     if not html_source:
         return html_source
 
     if not fromstring and tostring:
         return html_source
 
     try:
         doc = lxml.html.fromstring(html_source)
     except Exception:
         return html_source
 
     for el in doc.cssselect('img, video'):
         src = el.attrib.get('src')
         if src:
             el.attrib['src'] = relative_path(src, server_paths['raw'])
 
     for el in doc.cssselect('a:not(.gfm)'):
         src = el.attrib.get('href')
         if src:
             raw_mode = el.attrib['href'].endswith('?raw=1')
             if raw_mode:
                 el.attrib['href'] = relative_path(src, server_paths['raw'])
             else:
                 el.attrib['href'] = relative_path(src, server_paths['standard'])
 
     return lxml.html.tostring(doc)
 
 
 def relative_path(path, request_path, is_repo_file=None):
     """
     relative link support, path is a rel path, and request_path is current
     server path (not absolute)
 
     e.g.
 
     path = '../logo.png'
     request_path= '/repo/files/path/file.md'
     produces: '/repo/files/logo.png'
     """
     # TODO(marcink): unicode/str support ?
     # maybe=> safe_unicode(urllib.quote(safe_str(final_path), '/:'))
 
     def dummy_check(p):
         return True  # assume default is a valid file path
 
     is_repo_file = is_repo_file or dummy_check
     if not path:
         return request_path
 
     path = safe_unicode(path)
     request_path = safe_unicode(request_path)
 
-    if path.startswith((u'data:', u'javascript:', u'#', u':')):
+    if path.startswith(('data:', 'javascript:', '#', ':')):
         # skip data, anchor, invalid links
         return path
 
     is_absolute = bool(urllib.parse.urlparse(path).netloc)
     if is_absolute:
         return path
 
     if not request_path:
         return path
 
-    if path.startswith(u'/'):
+    if path.startswith('/'):
         path = path[1:]
 
-    if path.startswith(u'./'):
+    if path.startswith('./'):
         path = path[2:]
 
     parts = request_path.split('/')
     # compute how deep we need to traverse the request_path
     depth = 0
 
     if is_repo_file(request_path):
         # if request path is a VALID file, we use a relative path with
         # one level up
         depth += 1
 
-    while path.startswith(u'../'):
+    while path.startswith('../'):
         depth += 1
         path = path[3:]
 
     if depth > 0:
         parts = parts[:-depth]
 
     parts.append(path)
-    final_path = u'/'.join(parts).lstrip(u'/')
+    final_path = '/'.join(parts).lstrip('/')
 
-    return u'/' + final_path
+    return '/' + final_path
 
 
 _cached_markdown_renderer = None
 
 
 def get_markdown_renderer(extensions, output_format):
     global _cached_markdown_renderer
 
     if _cached_markdown_renderer is None:
         _cached_markdown_renderer = markdown.Markdown(
             extensions=extensions,
             enable_attributes=False, output_format=output_format)
     return _cached_markdown_renderer
 
 
 _cached_markdown_renderer_flavored = None
 
 
 def get_markdown_renderer_flavored(extensions, output_format):
     global _cached_markdown_renderer_flavored
 
     if _cached_markdown_renderer_flavored is None:
         _cached_markdown_renderer_flavored = markdown.Markdown(
             extensions=extensions + [GithubFlavoredMarkdownExtension()],
             enable_attributes=False, output_format=output_format)
     return _cached_markdown_renderer_flavored
 
 
 class MarkupRenderer(object):
     RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
 
     MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
     RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
     JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
     PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
 
     URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                          r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
 
     MENTION_PAT = re.compile(MENTIONS_REGEX)
 
     extensions = ['markdown.extensions.codehilite', 'markdown.extensions.extra',
                   'markdown.extensions.def_list', 'markdown.extensions.sane_lists']
 
     output_format = 'html4'
 
     # extension together with weights. Lower is first means we control how
     # extensions are attached to readme names with those.
     PLAIN_EXTS = [
         # prefer no extension
         ('', 0),  # special case that renders READMES names without extension
         ('.text', 2), ('.TEXT', 2),
         ('.txt', 3), ('.TXT', 3)
     ]
 
     RST_EXTS = [
         ('.rst', 1), ('.rest', 1),
         ('.RST', 2), ('.REST', 2)
     ]
 
     MARKDOWN_EXTS = [
         ('.md', 1), ('.MD', 1),
         ('.mkdn', 2), ('.MKDN', 2),
         ('.mdown', 3), ('.MDOWN', 3),
         ('.markdown', 4), ('.MARKDOWN', 4)
     ]
 
     def _detect_renderer(self, source, filename=None):
         """
         runs detection of what renderer should be used for generating html
         from a markup language
 
         filename can be also explicitly a renderer name
 
         :param source:
         :param filename:
         """
 
         if MarkupRenderer.MARKDOWN_PAT.findall(filename):
             detected_renderer = 'markdown'
         elif MarkupRenderer.RST_PAT.findall(filename):
             detected_renderer = 'rst'
         elif MarkupRenderer.JUPYTER_PAT.findall(filename):
             detected_renderer = 'jupyter'
         elif MarkupRenderer.PLAIN_PAT.findall(filename):
             detected_renderer = 'plain'
         else:
             detected_renderer = 'plain'
 
         return getattr(MarkupRenderer, detected_renderer)
 
     @classmethod
     def bleach_clean(cls, text):
         from .bleach_whitelist import markdown_attrs, markdown_tags
         allowed_tags = markdown_tags
         allowed_attrs = markdown_attrs
 
         try:
             return bleach.clean(text, tags=allowed_tags, attributes=allowed_attrs)
         except Exception:
             return 'UNPARSEABLE TEXT'
 
     @classmethod
     def renderer_from_filename(cls, filename, exclude):
         """
         Detect renderer markdown/rst from filename and optionally use exclude
         list to remove some options. This is mostly used in helpers.
         Returns None when no renderer can be detected.
         """
         def _filter(elements):
             if isinstance(exclude, (list, tuple)):
                 return [x for x in elements if x not in exclude]
             return elements
 
         if filename.endswith(
                 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
             return 'markdown'
         if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
             return 'rst'
 
         return None
 
     def render(self, source, filename=None):
         """
         Renders a given filename using detected renderer
         it detects renderers based on file extension or mimetype.
         At last it will just do a simple html replacing new lines with <br/>
 
         :param file_name:
         :param source:
         """
 
         renderer = self._detect_renderer(source, filename)
         readme_data = renderer(source)
         return readme_data
 
     @classmethod
     def _flavored_markdown(cls, text):
         """
         Github style flavored markdown
 
         :param text:
         """
 
         # Extract pre blocks.
         extractions = {}
 
         def pre_extraction_callback(matchobj):
             digest = md5_safe(matchobj.group(0))
             extractions[digest] = matchobj.group(0)
             return "{gfm-extraction-%s}" % digest
         pattern = re.compile(r'<pre>.*?</pre>', re.MULTILINE | re.DOTALL)
         text = re.sub(pattern, pre_extraction_callback, text)
 
         # Prevent foo_bar_baz from ending up with an italic word in the middle.
         def italic_callback(matchobj):
             s = matchobj.group(0)
             if list(s).count('_') >= 2:
                 return s.replace('_', r'\_')
             return s
         text = re.sub(r'^(?! {4}|\t)\w+_\w+_\w[\w_]*', italic_callback, text)
 
         # Insert pre block extractions.
         def pre_insert_callback(matchobj):
             return '\n\n' + extractions[matchobj.group(1)]
         text = re.sub(r'\{gfm-extraction-([0-9a-f]{32})\}',
                       pre_insert_callback, text)
 
         return text
 
     @classmethod
     def urlify_text(cls, text):
         def url_func(match_obj):
             url_full = match_obj.groups()[0]
             return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
 
         return cls.URL_PAT.sub(url_func, text)
 
     @classmethod
     def convert_mentions(cls, text, mode):
         mention_pat = cls.MENTION_PAT
 
         def wrapp(match_obj):
             uname = match_obj.groups()[0]
             hovercard_url = "pyroutes.url('hovercard_username', {'username': '%s'});" % uname
 
             if mode == 'markdown':
                 tmpl = '<strong class="tooltip-hovercard" data-hovercard-alt="{uname}" data-hovercard-url="{hovercard_url}">@{uname}</strong>'
             elif mode == 'rst':
                 tmpl = ' **@{uname}** '
             else:
                 raise ValueError('mode must be rst or markdown')
 
             return tmpl.format(**{'uname': uname,
                                   'hovercard_url': hovercard_url})
 
         return mention_pat.sub(wrapp, text).strip()
 
     @classmethod
     def plain(cls, source, universal_newline=True, leading_newline=True):
         source = safe_unicode(source)
         if universal_newline:
             newline = '\n'
             source = newline.join(source.splitlines())
 
         rendered_source = cls.urlify_text(source)
         source = ''
         if leading_newline:
             source += '<br />'
         source += rendered_source.replace("\n", '<br />')
 
         rendered = cls.bleach_clean(source)
         return rendered
 
     @classmethod
     def markdown(cls, source, safe=True, flavored=True, mentions=False,
                  clean_html=True):
         """
         returns markdown rendered code cleaned by the bleach library
         """
 
         if flavored:
             markdown_renderer = get_markdown_renderer_flavored(
                 cls.extensions, cls.output_format)
         else:
             markdown_renderer = get_markdown_renderer(
                 cls.extensions, cls.output_format)
 
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='markdown')
             # we extracted mentions render with this using Mentions false
             return cls.markdown(mention_hl, safe=safe, flavored=flavored,
                                 mentions=False)
 
         source = safe_unicode(source)
 
         try:
             if flavored:
                 source = cls._flavored_markdown(source)
             rendered = markdown_renderer.convert(source)
         except Exception:
             log.exception('Error when rendering Markdown')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 rendered = cls.plain(source)
             else:
                 raise
 
         if clean_html:
             rendered = cls.bleach_clean(rendered)
         return rendered
 
     @classmethod
     def rst(cls, source, safe=True, mentions=False, clean_html=False):
         if mentions:
             mention_hl = cls.convert_mentions(source, mode='rst')
             # we extracted mentions render with this using Mentions false
             return cls.rst(mention_hl, safe=safe, mentions=False)
 
         source = safe_unicode(source)
         try:
             docutils_settings = dict(
                 [(alias, None) for alias in
                  cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
 
             docutils_settings.update({
                 'input_encoding': 'unicode',
                 'report_level': 4,
                 'syntax_highlight': 'short',
             })
 
             for k, v in docutils_settings.items():
                 directives.register_directive(k, v)
 
             parts = publish_parts(source=source,
                                   writer=RhodeCodeWriter(),
                                   settings_overrides=docutils_settings)
             rendered = parts["fragment"]
             if clean_html:
                 rendered = cls.bleach_clean(rendered)
             return parts['html_title'] + rendered
         except Exception:
             log.exception('Error when rendering RST')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 return cls.plain(source)
             else:
                 raise
 
     @classmethod
     def jupyter(cls, source, safe=True):
         from rhodecode.lib import helpers
 
         from traitlets.config import Config
         import nbformat
         from nbconvert import HTMLExporter
         from nbconvert.preprocessors import Preprocessor
 
         class CustomHTMLExporter(HTMLExporter):
             def _template_file_default(self):
                 return 'basic'
 
         class Sandbox(Preprocessor):
 
             def preprocess(self, nb, resources):
                 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
                 for cell in nb['cells']:
                     if not safe:
                         continue
 
                     if 'outputs' in cell:
                         for cell_output in cell['outputs']:
                             if 'data' in cell_output:
                                 if 'application/javascript' in cell_output['data']:
                                     cell_output['data']['text/plain'] = sandbox_text
                                     cell_output['data'].pop('application/javascript', None)
 
                     if 'source' in cell and cell['cell_type'] == 'markdown':
                         # sanitize similar like in markdown
                         cell['source'] = cls.bleach_clean(cell['source'])
 
                 return nb, resources
 
         def _sanitize_resources(input_resources):
             """
             Skip/sanitize some of the CSS generated and included in jupyter
             so it doesn't messes up UI so much
             """
 
             # TODO(marcink): probably we should replace this with whole custom
             # CSS set that doesn't screw up, but jupyter generated html has some
             # special markers, so it requires Custom HTML exporter template with
             # _default_template_path_default, to achieve that
 
             # strip the reset CSS
             input_resources[0] = input_resources[0][input_resources[0].find('/*! Source'):]
             return input_resources
 
         def as_html(notebook):
             conf = Config()
             conf.CustomHTMLExporter.preprocessors = [Sandbox]
             html_exporter = CustomHTMLExporter(config=conf)
 
             (body, resources) = html_exporter.from_notebook_node(notebook)
             header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
             js = MakoTemplate(r'''
             <!-- MathJax configuration -->
             <script type="text/x-mathjax-config">
             MathJax.Hub.Config({
                 jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
                 extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
                 TeX: {
                     extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
                 },
                 tex2jax: {
                     inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                     displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
                     processEscapes: true,
                     processEnvironments: true
                 },
                 // Center justify equations in code and markdown cells. Elsewhere
                 // we use CSS to left justify single line equations in code cells.
                 displayAlign: 'center',
                 "HTML-CSS": {
                     styles: {'.MathJax_Display': {"margin": 0}},
                     linebreaks: { automatic: true },
                     availableFonts: ["STIX", "TeX"]
                 },
                 showMathMenu: false
             });
             </script>
             <!-- End of MathJax configuration -->
             <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
             ''').render(h=helpers)
 
             css = MakoTemplate(r'''
             <link rel="stylesheet" type="text/css" href="${h.asset('css/style-ipython.css', ver=ver)}" media="screen"/>
             ''').render(h=helpers, ver='ver1')
 
             body = '\n'.join([header, css, js, body])
             return body, resources
 
         notebook = nbformat.reads(source, as_version=4)
         (body, resources) = as_html(notebook)
         return body
 
 
 class RstTemplateRenderer(object):
 
     def __init__(self):
         base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
         rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
         self.template_store = TemplateLookup(
             directories=rst_template_dirs,
             input_encoding='utf-8',
             imports=['from rhodecode.lib import helpers as h'])
 
     def _get_template(self, templatename):
         return self.template_store.get_template(templatename)
 
     def render(self, template_name, **kwargs):
         template = self._get_template(template_name)
         return template.render(**kwargs)

rhodecode/lib/middleware/simplegit.py

@@ -1,156 +1,156 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleGit middleware for handling git protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 import os
 import re
 import logging
 import urllib.parse
 
 import rhodecode
 from rhodecode.lib import utils
 from rhodecode.lib import utils2
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 GIT_PROTO_PAT = re.compile(
     r'^/(.+)/(info/refs|info/lfs/(.+)|git-upload-pack|git-receive-pack)')
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def default_lfs_store():
     """
     Default lfs store location, it's consistent with Mercurials large file
     store which is in .cache/largefiles
     """
     from rhodecode.lib.vcs.backends.git import lfs_store
     user_home = os.path.expanduser("~")
     return lfs_store(user_home)
 
 
 class SimpleGit(simplevcs.SimpleVCS):
 
     SCM = 'git'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
         # for GIT LFS, and bare format strip .git suffix from names
         if repo_name.endswith('.git'):
             repo_name = repo_name[:-4]
         return repo_name
 
     def _get_lfs_action(self, path, request_method):
         """
         return an action based on LFS requests type.
         Those routes are handled inside vcsserver app.
 
         batch           -> POST to /info/lfs/objects/batch  => PUSH/PULL
                         batch is based on the `operation.
                         that could be download or upload, but those are only
                         instructions to fetch so we return pull always
 
         download        -> GET  to /info/lfs/{oid}          => PULL
         upload          -> PUT  to /info/lfs/{oid}          => PUSH
 
         verification    -> POST to /info/lfs/verify         => PULL
 
         """
 
         match_obj = GIT_LFS_PROTO_PAT.match(path)
         _parts = match_obj.groups()
         repo_name, path, operation = _parts
         log.debug(
             'LFS: detecting operation based on following '
             'data: %s, req_method:%s', _parts, request_method)
 
         if operation == 'verify':
             return 'pull'
         elif operation == 'objects/batch':
             # batch sends back instructions for API to dl/upl we report it
             # as pull
             if request_method == 'POST':
                 return 'pull'
 
         elif operation:
             # probably a OID, upload  is PUT, download a GET
             if request_method == 'GET':
                 return 'pull'
             else:
                 return 'push'
 
         # if default not found require push, as action
         return 'push'
 
     _ACTION_MAPPING = {
         'git-receive-pack': 'push',
         'git-upload-pack': 'pull',
     }
 
     def _get_action(self, environ):
         """
         Maps git request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'pull' to be safe.
 
         :param environ:
         """
         path = environ['PATH_INFO']
 
         if path.endswith('/info/refs'):
-            query = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING'])
+            query = urllib.parse.parse_qs(environ['QUERY_STRING'])
             service_cmd = query.get('service', [''])[0]
             return self._ACTION_MAPPING.get(service_cmd, 'pull')
 
         elif GIT_LFS_PROTO_PAT.match(environ['PATH_INFO']):
             return self._get_lfs_action(
                 environ['PATH_INFO'], environ['REQUEST_METHOD'])
 
         elif path.endswith('/git-receive-pack'):
             return 'push'
         elif path.endswith('/git-upload-pack'):
             return 'pull'
 
         return 'pull'
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_git_wsgi_app(
             repo_path, repo_name, config)
 
     def _create_config(self, extras, repo_name, scheme='http'):
         extras['git_update_server_info'] = utils2.str2bool(
             rhodecode.CONFIG.get('git_update_server_info'))
 
         config = utils.make_db_config(repo=repo_name)
         custom_store = config.get('vcs_git_lfs', 'store_location')
 
         extras['git_lfs_enabled'] = utils2.str2bool(
             config.get('vcs_git_lfs', 'enabled'))
         extras['git_lfs_store_path'] = custom_store or default_lfs_store()
         extras['git_lfs_http_scheme'] = scheme
         return extras

rhodecode/lib/middleware/simplehg.py

@@ -1,160 +1,159 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleHG middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 """
 
 import logging
 import urllib.parse
 import urllib.request, urllib.parse, urllib.error
 
 from rhodecode.lib import utils
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.middleware import simplevcs
 
 log = logging.getLogger(__name__)
 
 
 class SimpleHg(simplevcs.SimpleVCS):
 
     SCM = 'hg'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = environ['PATH_INFO']
         if repo_name and repo_name.startswith('/'):
             # remove only the first leading /
             repo_name = repo_name[1:]
         return repo_name.rstrip('/')
 
     _ACTION_MAPPING = {
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
         'getbundle': 'pull',
         'stream_out': 'pull',
         'listkeys': 'pull',
         'between': 'pull',
         'branchmap': 'pull',
         'branches': 'pull',
         'clonebundles': 'pull',
         'capabilities': 'pull',
         'debugwireargs': 'pull',
         'heads': 'pull',
         'lookup': 'pull',
         'hello': 'pull',
         'known': 'pull',
 
         # largefiles
         'putlfile': 'push',
         'getlfile': 'pull',
         'statlfile': 'pull',
         'lheads': 'pull',
 
         # evolve
         'evoext_obshashrange_v1': 'pull',
         'evoext_obshash': 'pull',
         'evoext_obshash1': 'pull',
 
         'unbundle': 'push',
         'pushkey': 'push',
     }
 
     @classmethod
     def _get_xarg_headers(cls, environ):
         i = 1
         chunks = []  # gather chunks stored in multiple 'hgarg_N'
         while True:
             head = environ.get('HTTP_X_HGARG_{}'.format(i))
             if not head:
                 break
             i += 1
             chunks.append(urllib.parse.unquote_plus(head))
         full_arg = ''.join(chunks)
         pref = 'cmds='
         if full_arg.startswith(pref):
             # strip the cmds= header defining our batch commands
             full_arg = full_arg[len(pref):]
         cmds = full_arg.split(';')
         return cmds
 
     @classmethod
     def _get_batch_cmd(cls, environ):
         """
         Handle batch command send commands. Those are ';' separated commands
         sent by batch command that server needs to execute. We need to extract
         those, and map them to our ACTION_MAPPING to get all push/pull commands
         specified in the batch
         """
         default = 'push'
         batch_cmds = []
         try:
             cmds = cls._get_xarg_headers(environ)
             for pair in cmds:
                 parts = pair.split(' ', 1)
                 if len(parts) != 2:
                     continue
                 # entry should be in a format `key ARGS`
                 cmd, args = parts
                 action = cls._ACTION_MAPPING.get(cmd, default)
                 batch_cmds.append(action)
         except Exception:
             log.exception('Failed to extract batch commands operations')
 
         # in case we failed, (e.g malformed data) assume it's PUSH sub-command
         # for safety
         return batch_cmds or [default]
 
     def _get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'push' to be safe.
 
         :param environ:
         """
         default = 'push'
-        query = urllib.parse.urlparse.parse_qs(environ['QUERY_STRING'],
-                                  keep_blank_values=True)
+        query = urllib.parse.parse_qs(environ['QUERY_STRING'], keep_blank_values=True)
 
         if 'cmd' in query:
             cmd = query['cmd'][0]
             if cmd == 'batch':
                 cmds = self._get_batch_cmd(environ)
                 if 'push' in cmds:
                     return 'push'
                 else:
                     return 'pull'
             return self._ACTION_MAPPING.get(cmd, default)
 
         return default
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
 
     def _create_config(self, extras, repo_name, scheme='http'):
         config = utils.make_db_config(repo=repo_name)
         config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
         return config.serialize()

rhodecode/lib/middleware/simplevcs.py

@@ -1,679 +1,679 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
+import io
 import logging
 import importlib
 from functools import wraps
-from io import StringIO
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = next(gen)
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
-        'pull-request/(?P<pr_id>\d+)/'  # pull request
+        'pull-request/(?P<pr_id>\\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
         realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
 
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = user_id
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
             data = environ['wsgi.input'].read()
-            environ['wsgi.input'] = StringIO(data)
+            environ['wsgi.input'] = io.StringIO(data)
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             direct_calls = True
 
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST, use_direct_calls=direct_calls, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']