feat(ssh-wrapper-speedup): major rewrite of code to address imports problem with ssh-wrapper-v2

- use bootstrapped settings rather than config
- use more code split to make sure we don't import heavy code

rhodecode/config/config_maker.py

@@ -0,0 +1,198 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import tempfile
+import logging
+
+from pyramid.settings import asbool
+
+from rhodecode.config.settings_maker import SettingsMaker
+from rhodecode.config import utils as config_utils
+
+log = logging.getLogger(__name__)
+
+
+def sanitize_settings_and_apply_defaults(global_config, settings):
+    """
+    Applies settings defaults and does all type conversion.
+
+    We would move all settings parsing and preparation into this place, so that
+    we have only one place left which deals with this part. The remaining parts
+    of the application would start to rely fully on well-prepared settings.
+
+    This piece would later be split up per topic to avoid a big fat monster
+    function.
+    """
+    jn = os.path.join
+
+    global_settings_maker = SettingsMaker(global_config)
+    global_settings_maker.make_setting('debug', default=False, parser='bool')
+    debug_enabled = asbool(global_config.get('debug'))
+
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=False,
+        parser='bool')
+
+    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
+    settings_maker.make_setting('rhodecode.edition_id', 'CE')
+
+    if 'mako.default_filters' not in settings:
+        # set custom default filters if we don't have it defined
+        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
+        settings['mako.default_filters'] = 'h_filter'
+
+    if 'mako.directories' not in settings:
+        mako_directories = settings.setdefault('mako.directories', [
+            # Base templates of the original application
+            'rhodecode:templates',
+        ])
+        log.debug(
+            "Using the following Mako template directories: %s",
+            mako_directories)
+
+    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
+    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
+        raw_url = settings['beaker.session.url']
+        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
+            settings['beaker.session.url'] = 'redis://' + raw_url
+
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
+
+    # TODO: johbo: Re-think this, usually the call to config.include
+    # should allow to pass in a prefix.
+    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
+
+    # Sanitize generic settings.
+    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
+    settings_maker.make_setting('is_test', False, parser='bool')
+    settings_maker.make_setting('gzip_responses', False, parser='bool')
+
+    # statsd
+    settings_maker.make_setting('statsd.enabled', False, parser='bool')
+    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
+    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
+    settings_maker.make_setting('statsd.statsd_prefix', '')
+    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
+
+    settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.host', '*')
+    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
+    settings_maker.make_setting('vcs.server', '')
+    settings_maker.make_setting('vcs.server.protocol', 'http')
+    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
+    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
+    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
+    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
+    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
+    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+
+    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
+
+    # Support legacy values of vcs.scm_app_implementation. Legacy
+    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
+    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
+    scm_app_impl = settings['vcs.scm_app_implementation']
+    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
+        settings['vcs.scm_app_implementation'] = 'http'
+
+    settings_maker.make_setting('appenlight', False, parser='bool')
+
+    temp_store = tempfile.gettempdir()
+    tmp_cache_dir = jn(temp_store, 'rc_cache')
+
+    # save default, cache dir, and use it for all backends later.
+    default_cache_dir = settings_maker.make_setting(
+        'cache_dir',
+        default=tmp_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
+
+    # exception store cache
+    settings_maker.make_setting(
+        'exception_tracker.store_path',
+        default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    settings_maker.make_setting(
+        'celerybeat-schedule.path',
+        default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
+        parser='file:ensured'
+    )
+
+    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
+    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
+
+    # sessions, ensure file since no-value is memory
+    settings_maker.make_setting('beaker.session.type', 'file')
+    settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
+
+    # cache_general
+    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
+    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
+
+    # cache_perms
+    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
+
+    # cache_repo
+    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
+
+    # cache_license
+    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
+
+    # cache_repo_longterm memory, 96H
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
+
+    # sql_cache_short
+    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
+    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
+
+    # archive_cache
+    settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
+
+    settings_maker.env_expand()
+
+    # configure instance id
+    config_utils.set_instance_id(settings)
+
+    return settings

rhodecode/lib/api_utils.py

@@ -0,0 +1,38 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import urllib.parse
+
+from rhodecode.lib.vcs import CurlSession
+from rhodecode.lib.ext_json import json
+
+
+def call_service_api(service_api_host, service_api_token, api_url, payload):
+
+    payload.update({
+        'id': 'service',
+        'auth_token': service_api_token
+    })
+
+    service_api_url = urllib.parse.urljoin(service_api_host, api_url)
+    response = CurlSession().post(service_api_url, json.dumps(payload))
+
+    if response.status_code != 200:
+        raise Exception(f"Service API at {service_api_url} responded with error: {response.status_code}")
+
+    return json.loads(response.content)['result']

rhodecode/lib/config_utils.py

@@ -0,0 +1,40 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+
+
+def get_config(ini_path, **kwargs):
+    import configparser
+    parser = configparser.ConfigParser(**kwargs)
+    parser.read(ini_path)
+    return parser
+
+
+def get_app_config_lightweight(ini_path):
+    parser = get_config(ini_path)
+    parser.set('app:main', 'here', os.getcwd())
+    parser.set('app:main', '__file__', ini_path)
+    return dict(parser.items('app:main'))
+
+
+def get_app_config(ini_path):
+    """
+    This loads the app context and provides a heavy type iniliaziation of config
+    """
+    from paste.deploy.loadwsgi import appconfig
+    return appconfig(f'config:{ini_path}', relative_to=os.getcwd())

rhodecode/lib/hook_daemon/__init__.py

@@ -0,0 +1,17 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/lib/hook_daemon/base.py

@@ -0,0 +1,115 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import time
+import logging
+import tempfile
+
+from rhodecode.lib.config_utils import get_config
+from rhodecode.lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class BaseHooksCallbackDaemon:
+    """
+    Basic context manager for actions that don't require some extra
+    """
+    def __init__(self):
+        pass
+
+    def __enter__(self):
+        log.debug('Running `%s` callback daemon', self.__class__.__name__)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
+
+
+class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
+
+    def __init__(self, module):
+        super().__init__()
+        self.hooks_module = module
+
+
+def get_txn_id_data_path(txn_id):
+    import rhodecode
+
+    root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
+    final_dir = os.path.join(root, 'svn_txn_id')
+
+    if not os.path.isdir(final_dir):
+        os.makedirs(final_dir)
+    return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
+
+
+def store_txn_id_data(txn_id, data_dict):
+    if not txn_id:
+        log.warning('Cannot store txn_id because it is empty')
+        return
+
+    path = get_txn_id_data_path(txn_id)
+    try:
+        with open(path, 'wb') as f:
+            f.write(json.dumps(data_dict))
+    except Exception:
+        log.exception('Failed to write txn_id metadata')
+
+
+def get_txn_id_from_store(txn_id):
+    """
+    Reads txn_id from store and if present returns the data for callback manager
+    """
+    path = get_txn_id_data_path(txn_id)
+    try:
+        with open(path, 'rb') as f:
+            return json.loads(f.read())
+    except Exception:
+        return {}
+
+
+def prepare_callback_daemon(extras, protocol, host, txn_id=None):
+    txn_details = get_txn_id_from_store(txn_id)
+    port = txn_details.get('port', 0)
+    match protocol:
+        case 'http':
+            from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
+            callback_daemon = HttpHooksCallbackDaemon(
+                txn_id=txn_id, host=host, port=port)
+        case 'celery':
+            from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
+            callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
+        case 'local':
+            from rhodecode.lib.hook_daemon.hook_module import Hooks
+            callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
+        case _:
+            log.error('Unsupported callback daemon protocol "%s"', protocol)
+            raise Exception('Unsupported callback daemon protocol.')
+
+    extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
+    extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
+    extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
+    extras['hooks_protocol'] = protocol
+    extras['time'] = time.time()
+
+    # register txn_id
+    extras['txn_id'] = txn_id
+    log.debug('Prepared a callback daemon: %s',
+              callback_daemon.__class__.__name__)
+    return callback_daemon, extras

rhodecode/lib/hook_daemon/celery_hooks_deamon.py

@@ -0,0 +1,30 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
+
+
+class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
+    """
+    Context manger for achieving a compatibility with celery backend
+    """
+
+    def __init__(self, config):
+        # TODO: replace this with settings bootstrapped...
+        self.task_queue = config.get('app:main', 'celery.broker_url')
+        self.task_backend = config.get('app:main', 'celery.result_backend')

rhodecode/lib/hook_daemon/hook_module.py

@@ -0,0 +1,104 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import traceback
+
+from rhodecode.model import meta
+
+from rhodecode.lib import hooks_base
+from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
+from rhodecode.lib.utils2 import AttributeDict
+
+log = logging.getLogger(__name__)
+
+
+class Hooks(object):
+    """
+    Exposes the hooks for remote callbacks
+    """
+    def __init__(self, request=None, log_prefix=''):
+        self.log_prefix = log_prefix
+        self.request = request
+
+    def repo_size(self, extras):
+        log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.repo_size, extras)
+
+    def pre_pull(self, extras):
+        log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_pull, extras)
+
+    def post_pull(self, extras):
+        log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_pull, extras)
+
+    def pre_push(self, extras):
+        log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_push, extras)
+
+    def post_push(self, extras):
+        log.debug("%sCalled post_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_push, extras)
+
+    def _call_hook(self, hook, extras):
+        extras = AttributeDict(extras)
+        _server_url = extras['server_url']
+
+        extras.request = self.request
+
+        try:
+            result = hook(extras)
+            if result is None:
+                raise Exception(f'Failed to obtain hook result from func: {hook}')
+        except HTTPBranchProtected as handled_error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            result = AttributeDict({
+                'status': handled_error.code,
+                'output': handled_error.explanation
+            })
+        except (HTTPLockedRC, Exception) as error:
+            # locked needs different handling since we need to also
+            # handle PULL operations
+            exc_tb = ''
+            if not isinstance(error, HTTPLockedRC):
+                exc_tb = traceback.format_exc()
+                log.exception('%sException when handling hook %s', self.log_prefix, hook)
+            error_args = error.args
+            return {
+                'status': 128,
+                'output': '',
+                'exception': type(error).__name__,
+                'exception_traceback': exc_tb,
+                'exception_args': error_args,
+            }
+        finally:
+            meta.Session.remove()
+
+        log.debug('%sGot hook call response %s', self.log_prefix, result)
+        return {
+            'status': result.status,
+            'output': result.output,
+        }
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass

rhodecode/lib/hook_daemon/http_hooks_deamon.py

@@ -0,0 +1,280 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import logging
+import traceback
+import threading
+import socket
+import msgpack
+import gevent
+
+from http.server import BaseHTTPRequestHandler
+from socketserver import TCPServer
+
+from rhodecode.model import meta
+from rhodecode.lib.ext_json import json
+from rhodecode.lib import rc_cache
+from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
+from rhodecode.lib.hook_daemon.hook_module import Hooks
+
+log = logging.getLogger(__name__)
+
+
+class HooksHttpHandler(BaseHTTPRequestHandler):
+
+    JSON_HOOKS_PROTO = 'json.v1'
+    MSGPACK_HOOKS_PROTO = 'msgpack.v1'
+    # starting with RhodeCode 5.0.0 MsgPack is the default, prior it used json
+    DEFAULT_HOOKS_PROTO = MSGPACK_HOOKS_PROTO
+
+    @classmethod
+    def serialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
+        if proto == cls.MSGPACK_HOOKS_PROTO:
+            return msgpack.packb(data)
+        return json.dumps(data)
+
+    @classmethod
+    def deserialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
+        if proto == cls.MSGPACK_HOOKS_PROTO:
+            return msgpack.unpackb(data)
+        return json.loads(data)
+
+    def do_POST(self):
+        hooks_proto, method, extras = self._read_request()
+        log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
+
+        txn_id = getattr(self.server, 'txn_id', None)
+        if txn_id:
+            log.debug('Computing TXN_ID based on `%s`:`%s`',
+                      extras['repository'], extras['txn_id'])
+            computed_txn_id = rc_cache.utils.compute_key_from_params(
+                extras['repository'], extras['txn_id'])
+            if txn_id != computed_txn_id:
+                raise Exception(
+                    'TXN ID fail: expected {} got {} instead'.format(
+                        txn_id, computed_txn_id))
+
+        request = getattr(self.server, 'request', None)
+        try:
+            hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
+            result = self._call_hook_method(hooks, method, extras)
+
+        except Exception as e:
+            exc_tb = traceback.format_exc()
+            result = {
+                'exception': e.__class__.__name__,
+                'exception_traceback': exc_tb,
+                'exception_args': e.args
+            }
+        self._write_response(hooks_proto, result)
+
+    def _read_request(self):
+        length = int(self.headers['Content-Length'])
+        # respect sent headers, fallback to OLD proto for compatability
+        hooks_proto = self.headers.get('rc-hooks-protocol') or self.JSON_HOOKS_PROTO
+        if hooks_proto == self.MSGPACK_HOOKS_PROTO:
+            # support for new vcsserver msgpack based protocol hooks
+            body = self.rfile.read(length)
+            data = self.deserialize_data(body)
+        else:
+            body = self.rfile.read(length)
+            data = self.deserialize_data(body)
+
+        return hooks_proto, data['method'], data['extras']
+
+    def _write_response(self, hooks_proto, result):
+        self.send_response(200)
+        if hooks_proto == self.MSGPACK_HOOKS_PROTO:
+            self.send_header("Content-type", "application/msgpack")
+            self.end_headers()
+            data = self.serialize_data(result)
+            self.wfile.write(data)
+        else:
+            self.send_header("Content-type", "text/json")
+            self.end_headers()
+            data = self.serialize_data(result)
+            self.wfile.write(data)
+
+    def _call_hook_method(self, hooks, method, extras):
+        try:
+            result = getattr(hooks, method)(extras)
+        finally:
+            meta.Session.remove()
+        return result
+
+    def log_message(self, format, *args):
+        """
+        This is an overridden method of BaseHTTPRequestHandler which logs using
+        a logging library instead of writing directly to stderr.
+        """
+
+        message = format % args
+
+        log.debug(
+            "HOOKS: client=%s - - [%s] %s", self.client_address,
+            self.log_date_time_string(), message)
+
+
+class ThreadedHookCallbackDaemon(object):
+
+    _callback_thread = None
+    _daemon = None
+    _done = False
+    use_gevent = False
+
+    def __init__(self, txn_id=None, host=None, port=None):
+        self._prepare(txn_id=txn_id, host=host, port=port)
+        if self.use_gevent:
+            self._run_func = self._run_gevent
+            self._stop_func = self._stop_gevent
+        else:
+            self._run_func = self._run
+            self._stop_func = self._stop
+
+    def __enter__(self):
+        log.debug('Running `%s` callback daemon', self.__class__.__name__)
+        self._run_func()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
+        self._stop_func()
+
+    def _prepare(self, txn_id=None, host=None, port=None):
+        raise NotImplementedError()
+
+    def _run(self):
+        raise NotImplementedError()
+
+    def _stop(self):
+        raise NotImplementedError()
+
+    def _run_gevent(self):
+        raise NotImplementedError()
+
+    def _stop_gevent(self):
+        raise NotImplementedError()
+
+
+class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
+    """
+    Context manager which will run a callback daemon in a background thread.
+    """
+
+    hooks_uri = None
+
+    # From Python docs: Polling reduces our responsiveness to a shutdown
+    # request and wastes cpu at all other times.
+    POLL_INTERVAL = 0.01
+
+    use_gevent = False
+
+    @property
+    def _hook_prefix(self):
+        return 'HOOKS: {} '.format(self.hooks_uri)
+
+    def get_hostname(self):
+        return socket.gethostname() or '127.0.0.1'
+
+    def get_available_port(self, min_port=20000, max_port=65535):
+        from rhodecode.lib.utils2 import get_available_port as _get_port
+        return _get_port(min_port, max_port)
+
+    def _prepare(self, txn_id=None, host=None, port=None):
+        from pyramid.threadlocal import get_current_request
+
+        if not host or host == "*":
+            host = self.get_hostname()
+        if not port:
+            port = self.get_available_port()
+
+        server_address = (host, port)
+        self.hooks_uri = '{}:{}'.format(host, port)
+        self.txn_id = txn_id
+        self._done = False
+
+        log.debug(
+            "%s Preparing HTTP callback daemon registering hook object: %s",
+            self._hook_prefix, HooksHttpHandler)
+
+        self._daemon = TCPServer(server_address, HooksHttpHandler)
+        # inject transaction_id for later verification
+        self._daemon.txn_id = self.txn_id
+
+        # pass the WEB app request into daemon
+        self._daemon.request = get_current_request()
+
+    def _run(self):
+        log.debug("Running thread-based loop of callback daemon in background")
+        callback_thread = threading.Thread(
+            target=self._daemon.serve_forever,
+            kwargs={'poll_interval': self.POLL_INTERVAL})
+        callback_thread.daemon = True
+        callback_thread.start()
+        self._callback_thread = callback_thread
+
+    def _run_gevent(self):
+        log.debug("Running gevent-based loop of callback daemon in background")
+        # create a new greenlet for the daemon's serve_forever method
+        callback_greenlet = gevent.spawn(
+            self._daemon.serve_forever,
+            poll_interval=self.POLL_INTERVAL)
+
+        # store reference to greenlet
+        self._callback_greenlet = callback_greenlet
+
+        # switch to this greenlet
+        gevent.sleep(0.01)
+
+    def _stop(self):
+        log.debug("Waiting for background thread to finish.")
+        self._daemon.shutdown()
+        self._callback_thread.join()
+        self._daemon = None
+        self._callback_thread = None
+        if self.txn_id:
+            txn_id_file = get_txn_id_data_path(self.txn_id)
+            log.debug('Cleaning up TXN ID %s', txn_id_file)
+            if os.path.isfile(txn_id_file):
+                os.remove(txn_id_file)
+
+        log.debug("Background thread done.")
+
+    def _stop_gevent(self):
+        log.debug("Waiting for background greenlet to finish.")
+
+        # if greenlet exists and is running
+        if self._callback_greenlet and not self._callback_greenlet.dead:
+            # shutdown daemon if it exists
+            if self._daemon:
+                self._daemon.shutdown()
+
+            # kill the greenlet
+            self._callback_greenlet.kill()
+
+        self._daemon = None
+        self._callback_greenlet = None
+
+        if self.txn_id:
+            txn_id_file = get_txn_id_data_path(self.txn_id)
+            log.debug('Cleaning up TXN ID %s', txn_id_file)
+            if os.path.isfile(txn_id_file):
+                os.remove(txn_id_file)
+
+        log.debug("Background greenlet done.")

rhodecode/apps/ssh_support/lib/backends/__init__.py

@@ -1,392 +1,417 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import datetime
-import configparser
 from sqlalchemy import Table
 
-from rhodecode.lib.utils import call_service_api
+from rhodecode.lib.api_utils import call_service_api
 from rhodecode.lib.utils2 import AttributeDict
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 from .hg import MercurialServer
 from .git import GitServer
 from .svn import SubversionServer
 log = logging.getLogger(__name__)
 
 
 class SshWrapper(object):
     hg_cmd_pat  = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$')
     git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
     svn_cmd_pat = re.compile(r'^svnserve -t')
 
     def __init__(self, command, connection_info, mode,
-                 user, user_id, key_id: int, shell, ini_path: str, env):
+                 user, user_id, key_id: int, shell, ini_path: str, settings, env):
         self.command = command
         self.connection_info = connection_info
         self.mode = mode
         self.username = user
         self.user_id = user_id
         self.key_id = key_id
         self.shell = shell
         self.ini_path = ini_path
         self.env = env
-
-        self.config = self.parse_config(ini_path)
+        self.settings = settings
         self.server_impl = None
 
-    def parse_config(self, config_path):
-        parser = configparser.ConfigParser()
-        parser.read(config_path)
-        return parser
-
     def update_key_access_time(self, key_id):
         from rhodecode.model.meta import raw_query_executor, Base
 
         table = Table('user_ssh_keys', Base.metadata, autoload=False)
         atime = datetime.datetime.utcnow()
         stmt = (
             table.update()
             .where(table.c.ssh_key_id == key_id)
             .values(accessed_on=atime)
             # no MySQL Support for .returning :((
             #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
         )
 
         res_count = None
         with raw_query_executor() as session:
             result = session.execute(stmt)
             if result.rowcount:
                 res_count = result.rowcount
 
         if res_count:
             log.debug('Update key id:`%s` access time', key_id)
 
     def get_user(self, user_id):
         user = AttributeDict()
         # lazy load db imports
         from rhodecode.model.db import User
         dbuser = User.get(user_id)
         if not dbuser:
             return None
         user.user_id = dbuser.user_id
         user.username = dbuser.username
         user.auth_user = dbuser.AuthUser()
         return user
 
     def get_connection_info(self):
         """
         connection_info
 
         Identifies the client and server ends of the connection.
         The variable contains four space-separated values: client IP address,
         client port number, server IP address, and server port number.
         """
         conn = dict(
             client_ip=None,
             client_port=None,
             server_ip=None,
             server_port=None,
         )
 
         info = self.connection_info.split(' ')
         if len(info) == 4:
             conn['client_ip'] = info[0]
             conn['client_port'] = info[1]
             conn['server_ip'] = info[2]
             conn['server_port'] = info[3]
 
         return conn
 
     def maybe_translate_repo_uid(self, repo_name):
         _org_name = repo_name
         if _org_name.startswith('_'):
             # remove format of _ID/subrepo
             _org_name = _org_name.split('/', 1)[0]
 
         if repo_name.startswith('_'):
             from rhodecode.model.repo import RepoModel
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 repo_name = by_id_match.repo_name
                 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
 
         return repo_name, _org_name
 
     def get_repo_details(self, mode):
         vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
         repo_name = None
 
         hg_match = self.hg_cmd_pat.match(self.command)
         if hg_match is not None:
             vcs_type = 'hg'
             repo_id = hg_match.group(1).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         git_match = self.git_cmd_pat.match(self.command)
         if git_match is not None:
             mode = git_match.group(1)
             vcs_type = 'git'
             repo_id = git_match.group(2).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         svn_match = self.svn_cmd_pat.match(self.command)
         if svn_match is not None:
             vcs_type = 'svn'
             # Repo name should be extracted from the input stream, we're unable to
             # extract it at this point in execution
             return vcs_type, repo_name, mode
 
         return vcs_type, repo_name, mode
 
     def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
+        # TODO: remove this once we have .ini defined access path...
+        from rhodecode.model.scm import ScmModel
+
         store = ScmModel().repos_path
 
         check_branch_perms = False
         detect_force_push = False
 
         if branch_permissions:
             check_branch_perms = True
             detect_force_push = True
 
         log.debug(
             'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
             vcs, mode, repo, check_branch_perms)
 
         # detect if we have to check branch permissions
         extras = {
             'detect_force_push': detect_force_push,
             'check_branch_perms': check_branch_perms,
             'config': self.ini_path
         }
 
         if vcs == 'hg':
             server = MercurialServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'git':
             server = GitServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, repo_mode=mode, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'svn':
             server = SubversionServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=None, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         else:
             raise Exception(f'Unrecognised VCS: {vcs}')
 
     def wrap(self):
         mode = self.mode
         username = self.username
         user_id = self.user_id
         key_id = self.key_id
         shell = self.shell
 
         scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
 
         log.debug(
             'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
             'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
             mode, username, user_id, shell, self.command,
             scm_detected, scm_mode, scm_repo)
 
         log.debug('SSH Connection info %s', self.get_connection_info())
 
         # update last access time for this key
         if key_id:
             self.update_key_access_time(key_id)
 
         if shell and self.command is None:
             log.info('Dropping to shell, no command given and shell is allowed')
             os.execl('/bin/bash', '-l')
             exit_code = 1
 
         elif scm_detected:
             user = self.get_user(user_id)
             if not user:
                 log.warning('User with id %s not found', user_id)
                 exit_code = -1
                 return exit_code
 
             auth_user = user.auth_user
             permissions = auth_user.permissions['repositories']
             repo_branch_permissions = auth_user.get_branch_permissions(scm_repo)
             try:
                 exit_code, is_updated = self.serve(
                     scm_detected, scm_repo, scm_mode, user, permissions,
                     repo_branch_permissions)
             except Exception:
                 log.exception('Error occurred during execution of SshWrapper')
                 exit_code = -1
 
         elif self.command is None and shell is False:
             log.error('No Command given.')
             exit_code = -1
 
         else:
             log.error('Unhandled Command: "%s" Aborting.', self.command)
             exit_code = -1
 
         return exit_code
 
 
 class SshWrapperStandalone(SshWrapper):
     """
     New version of SshWrapper designed to be depended only on service API
     """
     repos_path = None
+    service_api_host: str
+    service_api_token: str
+    api_url: str
+
+    def __init__(self, command, connection_info, mode,
+                 user, user_id, key_id: int, shell, ini_path: str, settings, env):
+
+        # validate our settings for making a standalone calls
+        try:
+            self.service_api_host = settings['app.service_api.host']
+            self.service_api_token = settings['app.service_api.token']
+        except KeyError:
+            raise ImproperlyConfiguredError(
+                "app.service_api.host or app.service_api.token are missing. "
+                "Please ensure that app.service_api.host and app.service_api.token are "
+                "defined inside of .ini configuration file."
+            )
+
+        try:
+            self.api_url = settings['rhodecode.api.url']
+        except KeyError:
+            raise ImproperlyConfiguredError(
+                "rhodecode.api.url is missing. "
+                "Please ensure that rhodecode.api.url is "
+                "defined inside of .ini configuration file."
+            )
+
+        super(SshWrapperStandalone, self).__init__(
+            command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
 
     @staticmethod
     def parse_user_related_data(user_data):
         user = AttributeDict()
         user.user_id = user_data['user_id']
         user.username = user_data['username']
         user.repo_permissions = user_data['repo_permissions']
         user.branch_permissions = user_data['branch_permissions']
         return user
 
     def wrap(self):
         mode = self.mode
         username = self.username
         user_id = self.user_id
         shell = self.shell
 
         scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
 
         log.debug(
             'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
             'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
             mode, username, user_id, shell, self.command,
             scm_detected, scm_mode, scm_repo)
 
         log.debug('SSH Connection info %s', self.get_connection_info())
 
         if shell and self.command is None:
             log.info('Dropping to shell, no command given and shell is allowed')
             os.execl('/bin/bash', '-l')
             exit_code = 1
 
         elif scm_detected:
-            data = call_service_api(self.ini_path, {
+            data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
                 "method": "service_get_data_for_ssh_wrapper",
                 "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
             })
             user = self.parse_user_related_data(data)
             if not user:
                 log.warning('User with id %s not found', user_id)
                 exit_code = -1
                 return exit_code
             self.repos_path = data['repos_path']
             permissions = user.repo_permissions
             repo_branch_permissions = user.branch_permissions
             try:
                 exit_code, is_updated = self.serve(
                     scm_detected, scm_repo, scm_mode, user, permissions,
                     repo_branch_permissions)
             except Exception:
                 log.exception('Error occurred during execution of SshWrapper')
                 exit_code = -1
 
         elif self.command is None and shell is False:
             log.error('No Command given.')
             exit_code = -1
 
         else:
             log.error('Unhandled Command: "%s" Aborting.', self.command)
             exit_code = -1
 
         return exit_code
 
     def maybe_translate_repo_uid(self, repo_name):
         _org_name = repo_name
         if _org_name.startswith('_'):
             _org_name = _org_name.split('/', 1)[0]
 
         if repo_name.startswith('_'):
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
-            by_id_match = call_service_api(self.ini_path, {
+            by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
                 'method': 'service_get_repo_name_by_id',
                 "args": {"repo_id": repo_name}
             })
             if by_id_match:
                 repo_name = by_id_match['repo_name']
                 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
 
         return repo_name, _org_name
 
     def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
         store = self.repos_path
 
         check_branch_perms = False
         detect_force_push = False
 
         if branch_permissions:
             check_branch_perms = True
             detect_force_push = True
 
         log.debug(
             'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
             vcs, mode, repo, check_branch_perms)
 
         # detect if we have to check branch permissions
         extras = {
             'detect_force_push': detect_force_push,
             'check_branch_perms': check_branch_perms,
             'config': self.ini_path
         }
 
         match vcs:
             case 'hg':
                 server = MercurialServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=repo, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case 'git':
                 server = GitServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=repo, repo_mode=mode, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case 'svn':
                 server = SubversionServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=None, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case _:
                 raise Exception(f'Unrecognised VCS: {vcs}')
         self.server_impl = server
         return server.run(tunnel_extras=extras)

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -1,174 +1,179 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
-from rhodecode.lib.utils import call_service_api
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.api_utils import call_service_api
 
 log = logging.getLogger(__name__)
 
 
-class VcsServer(object):
+class SSHVcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
+    settings = None  # parsed settings module
     write_perms = ['repository.admin', 'repository.write']
     read_perms = ['repository.read', 'repository.admin', 'repository.write']
 
-    def __init__(self, user, user_permissions, config, env):
+    def __init__(self, user, user_permissions, settings, env):
         self.user = user
         self.user_permissions = user_permissions
-        self.config = config
+        self.settings = settings
         self.env = env
         self.stdin = sys.stdin
 
         self.repo_name = None
         self.repo_mode = None
         self.store = ''
         self.ini_path = ''
         self.hooks_protocol = None
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         # Todo: Leave only "celery" case after transition.
         match self.hooks_protocol:
             case 'http':
+                from rhodecode.model.scm import ScmModel
                 ScmModel().mark_for_invalidation(repo_name)
             case 'celery':
-                call_service_api(self.ini_path, {
+                service_api_host = self.settings['app.service_api.host']
+                service_api_token = self.settings['app.service_api.token']
+                api_url = self.settings['rhodecode.api.url']
+
+                call_service_api(service_api_host, service_api_token, api_url, {
                     "method": "service_mark_for_invalidation",
                     "args": {"repo_name": repo_name}
                 })
 
     def has_write_perm(self):
         permission = self.user_permissions.get(self.repo_name)
         if permission in ['repository.write', 'repository.admin']:
             return True
 
         return False
 
     def _check_permissions(self, action):
         permission = self.user_permissions.get(self.repo_name)
         user_info = f'{self.user["user_id"]}:{self.user["username"]}'
         log.debug('permission for %s on %s are: %s',
                   user_info, self.repo_name, permission)
 
         if not permission:
             log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
                       user_info, self.repo_name)
             return -2
 
         if action == 'pull':
             if permission in self.read_perms:
                 log.info(
                     'READ Permissions for User "%s" detected to repo "%s"!',
                     user_info, self.repo_name)
                 return 0
         else:
             if permission in self.write_perms:
                 log.info(
                     'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
                     user_info, self.repo_name)
                 return 0
 
         log.error('Cannot properly fetch or verify user `%s` permissions. '
                   'Permissions: %s, vcs action: %s',
                   user_info, permission, action)
         return -2
 
     def update_environment(self, action, extras=None):
 
         scm_data = {
             'ip': os.environ['SSH_CLIENT'].split()[0],
             'username': self.user.username,
             'user_id': self.user.user_id,
             'action': action,
             'repository': self.repo_name,
             'scm': self.backend,
             'config': self.ini_path,
             'repo_store': self.store,
             'make_lock': None,
             'locked_by': [None, None],
             'server_url': None,
             'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
             'hooks': ['push', 'pull'],
-            'hooks_module': 'rhodecode.lib.hooks_daemon',
+            'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'is_shadow_repo': False,
             'detect_force_push': False,
             'check_branch_perms': False,
 
             'SSH': True,
             'SSH_PERMISSIONS': self.user_permissions.get(self.repo_name),
         }
         if extras:
             scm_data.update(extras)
         os.putenv("RC_SCM_DATA", json.dumps(scm_data))
         return scm_data
 
     def get_root_store(self):
         root_store = self.store
         if not root_store.endswith('/'):
             # always append trailing slash
             root_store = root_store + '/'
         return root_store
 
     def _handle_tunnel(self, extras):
         # pre-auth
         action = 'pull'
         exit_code = self._check_permissions(action)
         if exit_code:
             return exit_code, False
 
         req = self.env.get('request')
         if req:
             server_url = req.host_url + req.script_name
             extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
 
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
-        self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
+        self.hooks_protocol = self.settings['vcs.hooks.protocol']
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=self.hooks_protocol,
             host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             try:
                 return self._handle_tunnel(extras)
             finally:
                 log.debug('Running cleanup with cache invalidation')
                 if self.repo_name:
                     self._invalidate_cache(self.repo_name)

rhodecode/apps/ssh_support/lib/backends/git.py

@@ -1,88 +1,86 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import sys
 import logging
 import subprocess
 
 from vcsserver import hooks
-from .base import VcsServer
+from .base import SSHVcsServer
 
 log = logging.getLogger(__name__)
 
 
 class GitTunnelWrapper(object):
     process = None
 
     def __init__(self, server):
         self.server = server
         self.stdin = sys.stdin
         self.stdout = sys.stdout
 
     def create_hooks_env(self):
         pass
 
     def command(self):
         root = self.server.get_root_store()
         command = "cd {root}; {git_path} {mode} '{root}{repo_name}'".format(
             root=root, git_path=self.server.git_path,
             mode=self.server.repo_mode, repo_name=self.server.repo_name)
         log.debug("Final CMD: %s", command)
         return command
 
     def run(self, extras):
         action = "push" if self.server.repo_mode == "receive-pack" else "pull"
         exit_code = self.server._check_permissions(action)
         if exit_code:
             return exit_code
 
         scm_extras = self.server.update_environment(action=action, extras=extras)
 
         if action == "pull":
             hook_response = hooks.git_pre_pull(scm_extras)
             pre_pull_messages = hook_response.output
             sys.stdout.write(pre_pull_messages)
 
         self.create_hooks_env()
         result = subprocess.run(self.command(), shell=True)
         result = result.returncode
 
         # Upload-pack == clone
         if action == "pull":
             hook_response = hooks.git_post_pull(scm_extras)
             post_pull_messages = hook_response.output
             sys.stderr.write(post_pull_messages)
         return result
 
 
-class GitServer(VcsServer):
+class GitServer(SSHVcsServer):
     backend = 'git'
     repo_user_agent = 'git'
 
-    def __init__(self, store, ini_path, repo_name, repo_mode,
-                 user, user_permissions, config, env):
-        super().\
-            __init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
 
         self.store = store
         self.ini_path = ini_path
         self.repo_name = repo_name
-        self._path = self.git_path = config.get('app:main', 'ssh.executable.git')
+        self._path = self.git_path = settings['ssh.executable.git']
 
         self.repo_mode = repo_mode
         self.tunnel = GitTunnelWrapper(server=self)

rhodecode/apps/ssh_support/lib/backends/hg.py

@@ -1,155 +1,160 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 import tempfile
 import textwrap
 import collections
-from .base import VcsServer
-from rhodecode.lib.utils import call_service_api
-from rhodecode.model.db import RhodeCodeUi
-from rhodecode.model.settings import VcsSettingsModel
+
+from .base import SSHVcsServer
+
+from rhodecode.lib.api_utils import call_service_api
 
 log = logging.getLogger(__name__)
 
 
 class MercurialTunnelWrapper(object):
     process = None
 
     def __init__(self, server):
         self.server = server
         self.stdin = sys.stdin
         self.stdout = sys.stdout
         self.hooks_env_fd, self.hooks_env_path = tempfile.mkstemp(prefix='hgrc_rhodecode_')
 
     def create_hooks_env(self):
         repo_name = self.server.repo_name
         hg_flags = self.server.config_to_hgrc(repo_name)
 
         content = textwrap.dedent(
             '''
             # RhodeCode SSH hooks version=2.0.0
             {custom}
             '''
         ).format(custom='\n'.join(hg_flags))
 
         root = self.server.get_root_store()
         hgrc_custom = os.path.join(root, repo_name, '.hg', 'hgrc_rhodecode')
         hgrc_main = os.path.join(root, repo_name, '.hg', 'hgrc')
 
         # cleanup custom hgrc file
         if os.path.isfile(hgrc_custom):
             with open(hgrc_custom, 'wb') as f:
-                f.write('')
+                f.write(b'')
             log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
 
         # write temp
         with os.fdopen(self.hooks_env_fd, 'w') as hooks_env_file:
             hooks_env_file.write(content)
 
         return self.hooks_env_path
 
     def remove_configs(self):
         os.remove(self.hooks_env_path)
 
     def command(self, hgrc_path):
         root = self.server.get_root_store()
 
         command = (
             "cd {root}; HGRCPATH={hgrc} {hg_path} -R {root}{repo_name} "
             "serve --stdio".format(
                 root=root, hg_path=self.server.hg_path,
                 repo_name=self.server.repo_name, hgrc=hgrc_path))
         log.debug("Final CMD: %s", command)
         return command
 
     def run(self, extras):
         # at this point we cannot tell, we do further ACL checks
         # inside the hooks
         action = '?'
         # permissions are check via `pre_push_ssh_auth` hook
         self.server.update_environment(action=action, extras=extras)
         custom_hgrc_file = self.create_hooks_env()
 
         try:
             return os.system(self.command(custom_hgrc_file))
         finally:
             self.remove_configs()
 
 
-class MercurialServer(VcsServer):
+class MercurialServer(SSHVcsServer):
     backend = 'hg'
     repo_user_agent = 'mercurial'
     cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
 
-    def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
-        super().__init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
 
         self.store = store
         self.ini_path = ini_path
         self.repo_name = repo_name
-        self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
+        self._path = self.hg_path = settings['ssh.executable.hg']
         self.tunnel = MercurialTunnelWrapper(server=self)
 
     def config_to_hgrc(self, repo_name):
         # Todo: once transition is done only call to service api should exist
         if self.hooks_protocol == 'celery':
-            data = call_service_api(self.ini_path, {
+            service_api_host = self.settings['app.service_api.host']
+            service_api_token = self.settings['app.service_api.token']
+            api_url = self.settings['rhodecode.api.url']
+            data = call_service_api(service_api_host, service_api_token, api_url, {
                 "method": "service_config_to_hgrc",
                 "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
             })
             return data['flags']
-
-        ui_sections = collections.defaultdict(list)
-        ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
+        else:
+            from rhodecode.model.db import RhodeCodeUi
+            from rhodecode.model.settings import VcsSettingsModel
+            ui_sections = collections.defaultdict(list)
+            ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
 
-        # write default hooks
-        default_hooks = [
-            ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
-            ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
-            ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
+            # write default hooks
+            default_hooks = [
+                ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
+                ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
+                ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
 
-            ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
-            ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
-        ]
+                ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
+                ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
+            ]
 
-        for k, v in default_hooks:
-            ui_sections['hooks'].append((k, v))
+            for k, v in default_hooks:
+                ui_sections['hooks'].append((k, v))
 
-        for entry in ui:
-            if not entry.active:
-                continue
-            sec = entry.section
-            key = entry.key
-
-            if sec in self.cli_flags:
-                # we want only custom hooks, so we skip builtins
-                if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+            for entry in ui:
+                if not entry.active:
                     continue
+                sec = entry.section
+                key = entry.key
 
-                ui_sections[sec].append([key, entry.value])
+                if sec in self.cli_flags:
+                    # we want only custom hooks, so we skip builtins
+                    if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+                        continue
 
-        flags = []
-        for _sec, key_val in ui_sections.items():
-            flags.append(' ')
-            flags.append(f'[{_sec}]')
-            for key, val in key_val:
-                flags.append(f'{key}= {val}')
-        return flags
+                    ui_sections[sec].append([key, entry.value])
+
+            flags = []
+            for _sec, key_val in ui_sections.items():
+                flags.append(' ')
+                flags.append(f'[{_sec}]')
+                for key, val in key_val:
+                    flags.append(f'{key}= {val}')
+            return flags

rhodecode/apps/ssh_support/lib/backends/svn.py

@@ -1,256 +1,254 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import sys
 import logging
 import signal
 import tempfile
 from subprocess import Popen, PIPE
 import urllib.parse
 
-from .base import VcsServer
+from .base import SSHVcsServer
 
 log = logging.getLogger(__name__)
 
 
 class SubversionTunnelWrapper(object):
     process = None
 
     def __init__(self, server):
         self.server = server
         self.timeout = 30
         self.stdin = sys.stdin
         self.stdout = sys.stdout
         self.svn_conf_fd, self.svn_conf_path = tempfile.mkstemp()
         self.hooks_env_fd, self.hooks_env_path = tempfile.mkstemp()
 
         self.read_only = True  # flag that we set to make the hooks readonly
 
     def create_svn_config(self):
         content = (
             '[general]\n'
             'hooks-env = {}\n').format(self.hooks_env_path)
         with os.fdopen(self.svn_conf_fd, 'w') as config_file:
             config_file.write(content)
 
     def create_hooks_env(self):
         content = (
             '[default]\n'
             'LANG = en_US.UTF-8\n')
         if self.read_only:
             content += 'SSH_READ_ONLY = 1\n'
         with os.fdopen(self.hooks_env_fd, 'w') as hooks_env_file:
             hooks_env_file.write(content)
 
     def remove_configs(self):
         os.remove(self.svn_conf_path)
         os.remove(self.hooks_env_path)
 
     def command(self):
         root = self.server.get_root_store()
         username = self.server.user.username
 
         command = [
             self.server.svn_path, '-t',
             '--config-file', self.svn_conf_path,
             '--tunnel-user', username,
             '-r', root]
         log.debug("Final CMD: %s", ' '.join(command))
         return command
 
     def start(self):
         command = self.command()
         self.process = Popen(' '.join(command), stdin=PIPE, shell=True)
 
     def sync(self):
         while self.process.poll() is None:
             next_byte = self.stdin.read(1)
             if not next_byte:
                 break
             self.process.stdin.write(next_byte)
         self.remove_configs()
 
     @property
     def return_code(self):
         return self.process.returncode
 
     def get_first_client_response(self):
         signal.signal(signal.SIGALRM, self.interrupt)
         signal.alarm(self.timeout)
         first_response = self._read_first_client_response()
         signal.alarm(0)
         return (self._parse_first_client_response(first_response)
                 if first_response else None)
 
     def patch_first_client_response(self, response, **kwargs):
         self.create_hooks_env()
         data = response.copy()
         data.update(kwargs)
         data['url'] = self._svn_string(data['url'])
         data['ra_client'] = self._svn_string(data['ra_client'])
         data['client'] = data['client'] or ''
         buffer_ = (
             "( {version} ( {capabilities} ) {url}{ra_client}"
             "( {client}) ) ".format(**data))
         self.process.stdin.write(buffer_)
 
     def fail(self, message):
         print("( failure ( ( 210005 {message} 0: 0 ) ) )".format(
             message=self._svn_string(message)))
         self.remove_configs()
         self.process.kill()
         return 1
 
     def interrupt(self, signum, frame):
         self.fail("Exited by timeout")
 
     def _svn_string(self, str_):
         if not str_:
             return ''
         return f'{len(str_)}:{str_} '
 
     def _read_first_client_response(self):
         buffer_ = ""
         brackets_stack = []
         while True:
             next_byte = self.stdin.read(1)
             buffer_ += next_byte
             if next_byte == "(":
                 brackets_stack.append(next_byte)
             elif next_byte == ")":
                 brackets_stack.pop()
             elif next_byte == " " and not brackets_stack:
                 break
 
         return buffer_
 
     def _parse_first_client_response(self, buffer_):
         """
         According to the Subversion RA protocol, the first request
         should look like:
 
         ( version:number ( cap:word ... ) url:string ? ra-client:string
            ( ? client:string ) )
 
         Please check https://svn.apache.org/repos/asf/subversion/trunk/subversion/libsvn_ra_svn/protocol
         """
         version_re = r'(?P<version>\d+)'
         capabilities_re = r'\(\s(?P<capabilities>[\w\d\-\ ]+)\s\)'
         url_re = r'\d+\:(?P<url>[\W\w]+)'
         ra_client_re = r'(\d+\:(?P<ra_client>[\W\w]+)\s)'
         client_re = r'(\d+\:(?P<client>[\W\w]+)\s)*'
         regex = re.compile(
             r'^\(\s{version}\s{capabilities}\s{url}\s{ra_client}'
             r'\(\s{client}\)\s\)\s*$'.format(
                 version=version_re, capabilities=capabilities_re,
                 url=url_re, ra_client=ra_client_re, client=client_re))
         matcher = regex.match(buffer_)
 
         return matcher.groupdict() if matcher else None
 
     def _match_repo_name(self, url):
         """
         Given an server url, try to match it against ALL known repository names.
         This handles a tricky SVN case for SSH and subdir commits.
         E.g if our repo name is my-svn-repo, a svn commit on file in a subdir would
         result in the url with this subdir added.
         """
         # case 1 direct match, we don't do any "heavy" lookups
         if url in self.server.user_permissions:
             return url
 
         log.debug('Extracting repository name from subdir path %s', url)
         # case 2 we check all permissions, and match closes possible case...
         # NOTE(dan): In this case we only know that url has a subdir parts, it's safe
         # to assume that it will have the repo name as prefix, we ensure the prefix
         # for similar repositories isn't matched by adding a /
         # e.g subgroup/repo-name/ and subgroup/repo-name-1/ would work correct.
         for repo_name in self.server.user_permissions:
             repo_name_prefix = repo_name + '/'
             if url.startswith(repo_name_prefix):
                 log.debug('Found prefix %s match, returning proper repository name',
                           repo_name_prefix)
                 return repo_name
 
         return
 
     def run(self, extras):
         action = 'pull'
         self.create_svn_config()
         self.start()
 
         first_response = self.get_first_client_response()
         if not first_response:
             return self.fail("Repository name cannot be extracted")
 
         url_parts = urllib.parse.urlparse(first_response['url'])
 
         self.server.repo_name = self._match_repo_name(url_parts.path.strip('/'))
 
         exit_code = self.server._check_permissions(action)
         if exit_code:
             return exit_code
 
         # set the readonly flag to False if we have proper permissions
         if self.server.has_write_perm():
             self.read_only = False
         self.server.update_environment(action=action, extras=extras)
 
         self.patch_first_client_response(first_response)
         self.sync()
         return self.return_code
 
 
-class SubversionServer(VcsServer):
+class SubversionServer(SSHVcsServer):
     backend = 'svn'
     repo_user_agent = 'svn'
 
-    def __init__(self, store, ini_path, repo_name,
-                 user, user_permissions, config, env):
-        super()\
-            .__init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
         self.store = store
         self.ini_path = ini_path
         # NOTE(dan): repo_name at this point is empty,
         # this is set later in .run() based from parsed input stream
         self.repo_name = repo_name
-        self._path = self.svn_path = config.get('app:main', 'ssh.executable.svn')
+        self._path = self.svn_path = settings['ssh.executable.svn']
 
         self.tunnel = SubversionTunnelWrapper(server=self)
 
     def _handle_tunnel(self, extras):
 
         # pre-auth
         action = 'pull'
         # Special case for SVN, we extract repo name at later stage
         # exit_code = self._check_permissions(action)
         # if exit_code:
         #     return exit_code, False
 
         req = self.env['request']
         server_url = req.host_url + req.script_name
         extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
 
         return exit_code, action == "push"
 
 

rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py

@@ -1,72 +1,72 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import time
 import logging
 
 import click
 
 from rhodecode.lib.pyramid_utils import bootstrap
 from rhodecode.lib.statsd_client import StatsdClient
 from .backends import SshWrapper
 from .utils import setup_custom_logging
 
 log = logging.getLogger(__name__)
 
 
-
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
     '--mode', '-m', required=False, default='auto',
     type=click.Choice(['auto', 'vcs', 'git', 'hg', 'svn', 'test']),
     help='mode of operation')
 @click.option('--user', help='Username for which the command will be executed')
 @click.option('--user-id', help='User ID for which the command will be executed')
 @click.option('--key-id', help='ID of the key from the database')
 @click.option('--shell', '-s', is_flag=True, help='Allow Shell')
 @click.option('--debug', is_flag=True, help='Enabled detailed output logging')
 def main(ini_path, mode, user, user_id, key_id, shell, debug):
     setup_custom_logging(ini_path, debug)
 
     command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
     if not command and mode not in ['test']:
         raise ValueError(
             'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
             'Please make sure this is set and available during execution '
             'of this script.')
     connection_info = os.environ.get('SSH_CONNECTION', '')
     time_start = time.time()
     with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
+        settings = env['registry'].settings
         statsd = StatsdClient.statsd
         try:
             ssh_wrapper = SshWrapper(
                 command, connection_info, mode,
-                user, user_id, key_id, shell, ini_path, env)
+                user, user_id, key_id, shell, ini_path, settings, env)
         except Exception:
             log.exception('Failed to execute SshWrapper')
             sys.exit(-5)
         return_code = ssh_wrapper.wrap()
         operation_took = time.time() - time_start
         if statsd:
             operation_took_ms = round(1000.0 * operation_took)
             statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
                           use_decimals=False)
         sys.exit(return_code)

rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py

@@ -1,70 +1,92 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+"""
+WARNING: be really carefully with changing ANY imports in this file
+# This script is to mean as really fast executable, doing some imports here that would yield an import chain change
+# can affect execution times...
+# This can be easily debugged using such command::
+# time PYTHONPROFILEIMPORTTIME=1 rc-ssh-wrapper-v2 --debug --mode=test .dev/dev.ini
+"""
+
 import os
 import sys
 import time
 import logging
 
 import click
 
+from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
 from rhodecode.lib.statsd_client import StatsdClient
+from rhodecode.lib.config_utils import get_app_config_lightweight
+
+from .utils import setup_custom_logging
 from .backends import SshWrapperStandalone
-from .utils import setup_custom_logging
 
 log = logging.getLogger(__name__)
 
 
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
     '--mode', '-m', required=False, default='auto',
     type=click.Choice(['auto', 'vcs', 'git', 'hg', 'svn', 'test']),
     help='mode of operation')
 @click.option('--user', help='Username for which the command will be executed')
 @click.option('--user-id', help='User ID for which the command will be executed')
 @click.option('--key-id', help='ID of the key from the database')
 @click.option('--shell', '-s', is_flag=True, help='Allow Shell')
 @click.option('--debug', is_flag=True, help='Enabled detailed output logging')
 def main(ini_path, mode, user, user_id, key_id, shell, debug):
+
+    time_start = time.time()
     setup_custom_logging(ini_path, debug)
 
     command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
     if not command and mode not in ['test']:
         raise ValueError(
             'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
             'Please make sure this is set and available during execution '
             'of this script.')
-    connection_info = os.environ.get('SSH_CONNECTION', '')
-    time_start = time.time()
-    env = {'RC_CMD_SSH_WRAPPER': '1'}
+
+    # initialize settings and get defaults
+    settings = get_app_config_lightweight(ini_path)
+    settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, settings)
+
+    # init and bootstrap StatsdClient
+    StatsdClient.setup(settings)
     statsd = StatsdClient.statsd
+
     try:
+        connection_info = os.environ.get('SSH_CONNECTION', '')
+        env = {'RC_CMD_SSH_WRAPPER': '1'}
         ssh_wrapper = SshWrapperStandalone(
             command, connection_info, mode,
-            user, user_id, key_id, shell, ini_path, env)
+            user, user_id, key_id, shell, ini_path, settings, env)
     except Exception:
         log.exception('Failed to execute SshWrapper')
         sys.exit(-5)
+
     return_code = ssh_wrapper.wrap()
     operation_took = time.time() - time_start
     if statsd:
         operation_took_ms = round(1000.0 * operation_took)
         statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
                       use_decimals=False)
+
     sys.exit(return_code)

rhodecode/apps/ssh_support/lib/utils.py

@@ -1,34 +1,34 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
-from pyramid.paster import setup_logging
 
 
 def setup_custom_logging(ini_path, debug):
     if debug:
+        from pyramid.paster import setup_logging  # Lazy import
         # enabled rhodecode.ini controlled logging setup
         setup_logging(ini_path)
     else:
         # configure logging in a mode that doesn't print anything.
         # in case of regularly configured logging it gets printed out back
         # to the client doing an SSH command.
         logger = logging.getLogger('')
         null = logging.NullHandler()
         # add the handler to the root logger
         logger.handlers = [null]

rhodecode/apps/ssh_support/tests/conftest.py

@@ -1,68 +1,71 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import pytest
 import configparser
 
 from rhodecode.apps.ssh_support.lib.ssh_wrapper_v1 import SshWrapper
 from rhodecode.lib.utils2 import AttributeDict
 
 
 @pytest.fixture()
 def dummy_conf_file(tmpdir):
     conf = configparser.ConfigParser()
     conf.add_section('app:main')
     conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
     conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
     conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
 
     f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
     with open(f_path, 'wt') as f:
         conf.write(f)
 
     return os.path.join(f_path)
 
 
 def plain_dummy_env():
     return {
         'request':
             AttributeDict(host_url='http://localhost', script_name='/')
     }
 
 
 @pytest.fixture()
 def dummy_env():
     return plain_dummy_env()
 
 
 def plain_dummy_user():
-    return AttributeDict(username='test_user')
+    return AttributeDict(
+        user_id=1,
+        username='test_user'
+    )
 
 
 @pytest.fixture()
 def dummy_user():
     return plain_dummy_user()
 
 
 @pytest.fixture()
 def ssh_wrapper(app, dummy_conf_file, dummy_env):
     conn_info = '127.0.0.1 22 10.0.0.1 443'
     return SshWrapper(
         'random command', conn_info, 'auto', 'admin', '1', key_id='1',
-        shell=False, ini_path=dummy_conf_file, env=dummy_env)
+        shell=False, ini_path=dummy_conf_file, settings={}, env=dummy_env)

rhodecode/apps/ssh_support/tests/test_server_git.py

@@ -1,153 +1,151 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.git import GitServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 from rhodecode.lib.ext_json import json
 
+
 class GitServerCreator(object):
     root = '/tmp/repo/path/'
     git_path = '/usr/local/bin/git'
     config_data = {
         'app:main': {
             'ssh.executable.git': git_path,
             'vcs.hooks.protocol': 'http',
         }
     }
     repo_name = 'test_git'
     repo_mode = 'receive-pack'
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'ini_path': '',
             'user': self.user,
             'repo_name': self.repo_name,
             'repo_mode': self.repo_mode,
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
         server = GitServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def git_server(app):
     return GitServerCreator()
 
 
 class TestGitServer(object):
 
     def test_command(self, git_server):
         server = git_server.create()
         expected_command = (
             'cd {root}; {git_path} {repo_mode} \'{root}{repo_name}\''.format(
                 root=git_server.root, git_path=git_server.git_path,
                 repo_mode=git_server.repo_mode, repo_name=git_server.repo_name)
         )
         assert expected_command == server.tunnel.command()
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test_git': 'repository.read'}, 'pull', 0),
         ({'test_git': 'repository.read'}, 'push', -2),
         ({'test_git': 'repository.write'}, 'push', 0),
         ({'test_git': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, git_server, permissions, action, code):
         server = git_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, value', [
         ({}, False),
         ({'test_git': 'repository.read'}, False),
         ({'test_git': 'repository.write'}, True),
         ({'test_git': 'repository.admin'}, True),
 
     ])
     def test_has_write_permissions(self, git_server, permissions, value):
         server = git_server.create(user_permissions=permissions)
         result = server.has_write_perm()
         assert result is value
 
     def test_run_returns_executes_command(self, git_server):
         server = git_server.create()
         from rhodecode.apps.ssh_support.lib.backends.git import GitTunnelWrapper
 
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(GitTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(GitTunnelWrapper, 'command', return_value='date'):
                 exit_code = server.run()
 
         assert exit_code == (0, False)
 
     @pytest.mark.parametrize(
         'repo_mode, action', [
             ['receive-pack', 'push'],
             ['upload-pack', 'pull']
         ])
     def test_update_environment(self, git_server, repo_mode, action):
         server = git_server.create(repo_mode=repo_mode)
         store = server.store
 
         with mock.patch('os.environ', {'SSH_CLIENT': '10.10.10.10 b'}):
             with mock.patch('os.putenv') as putenv_mock:
                 server.update_environment(action)
 
         expected_data = {
             'username': git_server.user.username,
             'user_id': git_server.user.user_id,
             'scm': 'git',
             'repository': git_server.repo_name,
             'make_lock': None,
             'action': action,
             'ip': '10.10.10.10',
             'locked_by': [None, None],
             'config': '',
             'repo_store': store,
             'server_url': None,
             'hooks': ['push', 'pull'],
             'is_shadow_repo': False,
-            'hooks_module': 'rhodecode.lib.hooks_daemon',
+            'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'check_branch_perms': False,
             'detect_force_push': False,
             'user_agent': u'git/ssh-user-agent',
             'SSH': True,
             'SSH_PERMISSIONS': 'repository.admin',
         }
         args, kwargs = putenv_mock.call_args
         assert json.loads(args[1]) == expected_data

rhodecode/apps/ssh_support/tests/test_server_hg.py

@@ -1,118 +1,115 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.hg import MercurialServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 
 
 class MercurialServerCreator(object):
     root = '/tmp/repo/path/'
     hg_path = '/usr/local/bin/hg'
 
     config_data = {
         'app:main': {
             'ssh.executable.hg': hg_path,
             'vcs.hooks.protocol': 'http',
         }
     }
     repo_name = 'test_hg'
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'ini_path': '',
             'user': self.user,
             'repo_name': self.repo_name,
             'user_permissions': {
                 'test_hg': 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
         server = MercurialServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def hg_server(app):
     return MercurialServerCreator()
 
 
 class TestMercurialServer(object):
 
     def test_command(self, hg_server, tmpdir):
         server = hg_server.create()
         custom_hgrc = os.path.join(str(tmpdir), 'hgrc')
         expected_command = (
             'cd {root}; HGRCPATH={custom_hgrc} {hg_path} -R {root}{repo_name} serve --stdio'.format(
                 root=hg_server.root, custom_hgrc=custom_hgrc, hg_path=hg_server.hg_path,
                 repo_name=hg_server.repo_name)
         )
         server_command = server.tunnel.command(custom_hgrc)
         assert expected_command == server_command
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test_hg': 'repository.read'}, 'pull', 0),
         ({'test_hg': 'repository.read'}, 'push', -2),
         ({'test_hg': 'repository.write'}, 'push', 0),
         ({'test_hg': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, hg_server, permissions, action, code):
         server = hg_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, value', [
         ({}, False),
         ({'test_hg': 'repository.read'}, False),
         ({'test_hg': 'repository.write'}, True),
         ({'test_hg': 'repository.admin'}, True),
 
     ])
     def test_has_write_permissions(self, hg_server, permissions, value):
         server = hg_server.create(user_permissions=permissions)
         result = server.has_write_perm()
         assert result is value
 
     def test_run_returns_executes_command(self, hg_server):
         server = hg_server.create()
         from rhodecode.apps.ssh_support.lib.backends.hg import MercurialTunnelWrapper
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(MercurialTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(MercurialTunnelWrapper, 'command', return_value='date'):
                 exit_code = server.run()
 
         assert exit_code == (0, False)
 
 
 

rhodecode/apps/ssh_support/tests/test_server_svn.py

@@ -1,205 +1,202 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.svn import SubversionServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 
 
 class SubversionServerCreator(object):
     root = '/tmp/repo/path/'
     svn_path = '/usr/local/bin/svnserve'
     config_data = {
         'app:main': {
             'ssh.executable.svn': svn_path,
             'vcs.hooks.protocol': 'http',
         }
     }
     repo_name = 'test-svn'
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'repo_name': self.repo_name,
             'ini_path': '',
             'user': self.user,
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
 
         parameters.update(kwargs)
         server = SubversionServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def svn_server(app):
     return SubversionServerCreator()
 
 
 class TestSubversionServer(object):
     def test_command(self, svn_server):
         server = svn_server.create()
         expected_command = [
             svn_server.svn_path, '-t',
             '--config-file', server.tunnel.svn_conf_path,
             '--tunnel-user', svn_server.user.username,
             '-r', svn_server.root
         ]
 
         assert expected_command == server.tunnel.command()
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test-svn': 'repository.read'}, 'pull', 0),
         ({'test-svn': 'repository.read'}, 'push', -2),
         ({'test-svn': 'repository.write'}, 'push', 0),
         ({'test-svn': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, svn_server, permissions, action, code):
         server = svn_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, access_paths, expected_match', [
         # not matched repository name
         ({
              'test-svn': ''
          }, ['test-svn-1', 'test-svn-1/subpath'],
          None),
 
         # exact match
         ({
              'test-svn': ''
          },
          ['test-svn'],
          'test-svn'),
 
         # subdir commits
         ({
              'test-svn': ''
          },
          ['test-svn/foo',
           'test-svn/foo/test-svn',
           'test-svn/trunk/development.txt',
           ],
          'test-svn'),
 
         # subgroups + similar patterns
         ({
              'test-svn': '',
              'test-svn-1': '',
              'test-svn-subgroup/test-svn': '',
 
          },
          ['test-svn-1',
           'test-svn-1/foo/test-svn',
           'test-svn-1/test-svn',
           ],
          'test-svn-1'),
 
         # subgroups + similar patterns
         ({
              'test-svn-1': '',
              'test-svn-10': '',
              'test-svn-100': '',
          },
          ['test-svn-10',
           'test-svn-10/foo/test-svn',
           'test-svn-10/test-svn',
           ],
          'test-svn-10'),
 
         # subgroups + similar patterns
         ({
              'name': '',
              'nameContains': '',
              'nameContainsThis': '',
          },
          ['nameContains',
           'nameContains/This',
           'nameContains/This/test-svn',
           ],
          'nameContains'),
 
         # subgroups + similar patterns
         ({
              'test-svn': '',
              'test-svn-1': '',
              'test-svn-subgroup/test-svn': '',
 
          },
          ['test-svn-subgroup/test-svn',
           'test-svn-subgroup/test-svn/foo/test-svn',
           'test-svn-subgroup/test-svn/trunk/example.txt',
           ],
          'test-svn-subgroup/test-svn'),
     ])
     def test_repo_extraction_on_subdir(self, svn_server, permissions, access_paths, expected_match):
         server = svn_server.create(user_permissions=permissions)
         for path in access_paths:
             repo_name = server.tunnel._match_repo_name(path)
             assert repo_name == expected_match
 
     def test_run_returns_executes_command(self, svn_server):
         server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(
                 SubversionTunnelWrapper, 'get_first_client_response',
                 return_value={'url': 'http://server/test-svn'}):
             with mock.patch.object(
                     SubversionTunnelWrapper, 'patch_first_client_response',
                     return_value=0):
                 with mock.patch.object(
                         SubversionTunnelWrapper, 'sync',
                         return_value=0):
                     with mock.patch.object(
                             SubversionTunnelWrapper, 'command',
                             return_value=['date']):
 
                         exit_code = server.run()
         # SVN has this differently configured, and we get in our mock env
         # None as return code
         assert exit_code == (None, False)
 
     def test_run_returns_executes_command_that_cannot_extract_repo_name(self, svn_server):
         server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
         with mock.patch.object(
                 SubversionTunnelWrapper, 'command',
                 return_value=['date']):
             with mock.patch.object(
                     SubversionTunnelWrapper, 'get_first_client_response',
                     return_value=None):
                     exit_code = server.run()
 
         assert exit_code == (1, False)

rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py

@@ -1,52 +1,48 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 
 class TestSSHWrapper(object):
 
     def test_serve_raises_an_exception_when_vcs_is_not_recognized(self, ssh_wrapper):
         with pytest.raises(Exception) as exc_info:
             ssh_wrapper.serve(
                 vcs='microsoft-tfs', repo='test-repo', mode=None, user='test',
                 permissions={}, branch_permissions={})
         assert str(exc_info.value) == 'Unrecognised VCS: microsoft-tfs'
 
-    def test_parse_config(self, ssh_wrapper):
-        config = ssh_wrapper.parse_config(ssh_wrapper.ini_path)
-        assert config
-
     def test_get_connection_info(self, ssh_wrapper):
         conn_info = ssh_wrapper.get_connection_info()
         assert {'client_ip': '127.0.0.1',
                 'client_port': '22',
                 'server_ip': '10.0.0.1',
                 'server_port': '443'} == conn_info
 
     @pytest.mark.parametrize('command, vcs', [
         ('xxx', None),
         ('svnserve -t', 'svn'),
         ('hg -R repo serve --stdio', 'hg'),
         ('git-receive-pack \'repo.git\'', 'git'),
 
     ])
     def test_get_repo_details(self, ssh_wrapper, command, vcs):
         ssh_wrapper.command = command
         vcs_type, repo_name, mode = ssh_wrapper.get_repo_details(mode='auto')
         assert vcs_type == vcs

rhodecode/config/middleware.py

@@ -1,637 +1,466 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import collections
-import tempfile
+
 import time
 import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
-from rhodecode import api
 from rhodecode.model import meta
 from rhodecode.config import patches
-from rhodecode.config import utils as config_utils
-from rhodecode.config.settings_maker import SettingsMaker
+
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
+from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception, format_exc
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, write_usage_data)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
     log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app
 
 
 def get_celery_config(settings):
     """
     Converts basic ini configuration into celery 4.X options
     """
 
     def key_converter(key_name):
         pref = 'celery.'
         if key_name.startswith(pref):
             return key_name[len(pref):].replace('.', '_').lower()
 
     def type_converter(parsed_key, value):
         # cast to int
         if value.isdigit():
             return int(value)
 
         # cast to bool
         if value.lower() in ['true', 'false', 'True', 'False']:
             return value.lower() == 'true'
         return value
 
     celery_config = {}
     for k, v in settings.items():
         pref = 'celery.'
         if k.startswith(pref):
             celery_config[key_converter(k)] = type_converter(key_converter(k), v)
 
     # TODO:rethink if we want to support celerybeat based file config, probably NOT
     # beat_config = {}
     # for section in parser.sections():
     #     if section.startswith('celerybeat:'):
     #         name = section.split(':', 1)[1]
     #         beat_config[name] = get_beat_config(parser, section)
 
     # final compose of settings
     celery_settings = {}
 
     if celery_config:
         celery_settings.update(celery_config)
     # if beat_config:
     #     celery_settings.update({'beat_schedule': beat_config})
 
     return celery_settings
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         traceback_info = format_exc(request.exc_info)
         log.error(
             'error occurred handling this request for path: %s, \n%s',
             request.path, traceback_info)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     debug_mode = rhodecode.ConfigGet().get_bool('debug')
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
     c.exception_debug = debug_mode
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
     if debug_mode:
         try:
             from rich.traceback import install
             install(show_locals=True)
             log.debug('Installing rich tracebacks...')
         except ImportError:
             pass
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     response.headers["X-RC-Exception-Id"] = str(c.exception_id)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
         exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:web",
                           f"http_code:{base_response.status_code}",
                           f"type:{exc_type}"])
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 ce_auth_resources = [
     'rhodecode.authentication.plugins.auth_crowd',
     'rhodecode.authentication.plugins.auth_headers',
     'rhodecode.authentication.plugins.auth_jasig_cas',
     'rhodecode.authentication.plugins.auth_ldap',
     'rhodecode.authentication.plugins.auth_pam',
     'rhodecode.authentication.plugins.auth_rhodecode',
     'rhodecode.authentication.plugins.auth_token',
 ]
 
 
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.lib.rc_cache.archive_cache')
 
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         log.debug('Starting config.include() calls')
         config.include('rhodecode.api.includeme')
         config.include('rhodecode.apps._base.includeme')
         config.include('rhodecode.apps._base.navigation.includeme')
         config.include('rhodecode.apps._base.subscribers.includeme')
         config.include('rhodecode.apps.hovercards.includeme')
         config.include('rhodecode.apps.ops.includeme')
         config.include('rhodecode.apps.channelstream.includeme')
         config.include('rhodecode.apps.file_store.includeme')
         config.include('rhodecode.apps.admin.includeme')
         config.include('rhodecode.apps.login.includeme')
         config.include('rhodecode.apps.home.includeme')
         config.include('rhodecode.apps.journal.includeme')
 
         config.include('rhodecode.apps.repository.includeme')
         config.include('rhodecode.apps.repo_group.includeme')
         config.include('rhodecode.apps.user_group.includeme')
         config.include('rhodecode.apps.search.includeme')
         config.include('rhodecode.apps.user_profile.includeme')
         config.include('rhodecode.apps.user_group_profile.includeme')
         config.include('rhodecode.apps.my_account.includeme')
         config.include('rhodecode.apps.gist.includeme')
 
         config.include('rhodecode.apps.svn_support.includeme')
         config.include('rhodecode.apps.ssh_support.includeme')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations.includeme')
         config.include('rhodecode.integrations.routes.includeme')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     settings['default_locale_name'] = settings.get('lang', 'en')
     config.add_translation_dirs('rhodecode:i18n/')
 
     # Add subscribers.
     if load_all:
         log.debug('Adding subscribers...')
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     log.debug('processing rhodecode.includes data...')
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         start = time.time()
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.get_engine().pool
             log.debug('sa pool status: %s', pool.status())
             total = time.time() - start
             log.debug('Request processing finalized: %.4fs', total)
 
     return pyramid_app_with_cleanup
-
-
-def sanitize_settings_and_apply_defaults(global_config, settings):
-    """
-    Applies settings defaults and does all type conversion.
-
-    We would move all settings parsing and preparation into this place, so that
-    we have only one place left which deals with this part. The remaining parts
-    of the application would start to rely fully on well prepared settings.
-
-    This piece would later be split up per topic to avoid a big fat monster
-    function.
-    """
-    jn = os.path.join
-
-    global_settings_maker = SettingsMaker(global_config)
-    global_settings_maker.make_setting('debug', default=False, parser='bool')
-    debug_enabled = asbool(global_config.get('debug'))
-
-    settings_maker = SettingsMaker(settings)
-
-    settings_maker.make_setting(
-        'logging.autoconfigure',
-        default=False,
-        parser='bool')
-
-    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
-    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
-
-    # Default includes, possible to change as a user
-    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
-    log.debug(
-        "Using the following pyramid.includes: %s",
-        pyramid_includes)
-
-    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
-    settings_maker.make_setting('rhodecode.edition_id', 'CE')
-
-    if 'mako.default_filters' not in settings:
-        # set custom default filters if we don't have it defined
-        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
-        settings['mako.default_filters'] = 'h_filter'
-
-    if 'mako.directories' not in settings:
-        mako_directories = settings.setdefault('mako.directories', [
-            # Base templates of the original application
-            'rhodecode:templates',
-        ])
-        log.debug(
-            "Using the following Mako template directories: %s",
-            mako_directories)
-
-    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
-    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
-        raw_url = settings['beaker.session.url']
-        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
-            settings['beaker.session.url'] = 'redis://' + raw_url
-
-    settings_maker.make_setting('__file__', global_config.get('__file__'))
-
-    # TODO: johbo: Re-think this, usually the call to config.include
-    # should allow to pass in a prefix.
-    settings_maker.make_setting('rhodecode.api.url', api.DEFAULT_URL)
-
-    # Sanitize generic settings.
-    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
-    settings_maker.make_setting('is_test', False, parser='bool')
-    settings_maker.make_setting('gzip_responses', False, parser='bool')
-
-    # statsd
-    settings_maker.make_setting('statsd.enabled', False, parser='bool')
-    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
-    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
-    settings_maker.make_setting('statsd.statsd_prefix', '')
-    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
-
-    settings_maker.make_setting('vcs.svn.compatible_version', '')
-    settings_maker.make_setting('vcs.hooks.protocol', 'http')
-    settings_maker.make_setting('vcs.hooks.host', '*')
-    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
-    settings_maker.make_setting('vcs.server', '')
-    settings_maker.make_setting('vcs.server.protocol', 'http')
-    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
-    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
-    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
-    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
-    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
-    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
-
-    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
-
-    # Support legacy values of vcs.scm_app_implementation. Legacy
-    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
-    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
-    scm_app_impl = settings['vcs.scm_app_implementation']
-    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
-        settings['vcs.scm_app_implementation'] = 'http'
-
-    settings_maker.make_setting('appenlight', False, parser='bool')
-
-    temp_store = tempfile.gettempdir()
-    tmp_cache_dir = jn(temp_store, 'rc_cache')
-
-    # save default, cache dir, and use it for all backends later.
-    default_cache_dir = settings_maker.make_setting(
-        'cache_dir',
-        default=tmp_cache_dir, default_when_empty=True,
-        parser='dir:ensured')
-
-    # exception store cache
-    settings_maker.make_setting(
-        'exception_tracker.store_path',
-        default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
-        parser='dir:ensured'
-    )
-
-    settings_maker.make_setting(
-        'celerybeat-schedule.path',
-        default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
-        parser='file:ensured'
-    )
-
-    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
-    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
-
-    # sessions, ensure file since no-value is memory
-    settings_maker.make_setting('beaker.session.type', 'file')
-    settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
-
-    # cache_general
-    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
-    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
-
-    # cache_perms
-    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
-    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
-
-    # cache_repo
-    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
-
-    # cache_license
-    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
-    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
-
-    # cache_repo_longterm memory, 96H
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
-
-    # sql_cache_short
-    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
-    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
-    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
-
-    # archive_cache
-    settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
-    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
-    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
-
-    settings_maker.env_expand()
-
-    # configure instance id
-    config_utils.set_instance_id(settings)
-
-    return settings

rhodecode/config/utils.py

@@ -1,117 +1,116 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import platform
 
-from rhodecode.model import init_model
-
 
 def configure_vcs(config):
     """
     Patch VCS config with some RhodeCode specific stuff
     """
     from rhodecode.lib.vcs import conf
     import rhodecode.lib.vcs.conf.settings
 
     conf.settings.BACKENDS = {
         'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
         'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
     conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
     conf.settings.HOOKS_HOST = config['vcs.hooks.host']
     conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
     conf.settings.ALIASES[:] = config['vcs.backends']
     conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version']
 
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
+    from rhodecode.model import init_model
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
 
 def initialize_test_environment(settings, test_env=None):
     if test_env is None:
         test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
 
     from rhodecode.lib.utils import (
         create_test_directory, create_test_database, create_test_repositories,
         create_test_index)
     from rhodecode.tests import TESTS_TMP_PATH
     from rhodecode.lib.vcs.backends.hg import largefiles_store
     from rhodecode.lib.vcs.backends.git import lfs_store
 
     # test repos
     if test_env:
         create_test_directory(TESTS_TMP_PATH)
         # large object stores
         create_test_directory(largefiles_store(TESTS_TMP_PATH))
         create_test_directory(lfs_store(TESTS_TMP_PATH))
 
         create_test_database(TESTS_TMP_PATH, settings)
         create_test_repositories(TESTS_TMP_PATH, settings)
         create_test_index(TESTS_TMP_PATH, settings)
 
 
 def get_vcs_server_protocol(config):
     return config['vcs.server.protocol']
 
 
 def set_instance_id(config):
     """
     Sets a dynamic generated config['instance_id'] if missing or '*'
     E.g instance_id = *cluster-1 or instance_id = *
     """
 
     config['instance_id'] = config.get('instance_id') or ''
     instance_id = config['instance_id']
     if instance_id.startswith('*') or not instance_id:
         prefix = instance_id.lstrip('*')
         _platform_id = platform.uname()[1] or 'instance'
         config['instance_id'] = '{prefix}uname:{platform}-pid:{pid}'.format(
             prefix=prefix,
             platform=_platform_id,
             pid=os.getpid())
 
 
 def get_default_user_id():
     DEFAULT_USER = 'default'
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
         result = session.execute(text("SELECT user_id from users where username = :uname"), {'uname': DEFAULT_USER})
         user_id = result.first()[0]
 
     return user_id
 
 
 def get_default_base_path():
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
         result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'"))
         base_path = result.first()[0]
 
     return base_path

rhodecode/lib/base.py

@@ -1,609 +1,609 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
 import base64
 
 import markupsafe
 import ipaddress
 
 import paste.httpheaders
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
 
 import rhodecode
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes)
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.str_utils import ascii_bytes, safe_int, safe_str
 from rhodecode.lib.type_utils import aslist, str2bool
 from rhodecode.lib.hash_utils import sha1
 from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_str(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
 
     def ip_filters(ip_):
         return _filter_port(_filter_proxy(ip_))
 
     ip = environ.get(proxy_key)
     if ip:
         return ip_filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return ip_filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return ip_filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False, check_branch_perms=False, detect_force_push=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     user = User.get_by_username(username)
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
     user_id = user.user_id
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     # NOTE(marcink): This should be also in sync with
     # rhodecode/apps/ssh_support/lib/backends/base.py:update_environment scm_data
     store = [x for x in ui_settings if x.key == '/']
     repo_store = ''
     if store:
         repo_store = store[0].value
 
     scm_data = {
         'ip': get_ip_addr(environ),
         'username': username,
         'user_id': user_id,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'repo_store': repo_store,
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
         'detect_force_push': detect_force_push,
         'check_branch_perms': check_branch_perms,
     }
     return scm_data
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
                  initial_call_detection=False, acl_repo_name=None, rc_realm=''):
         super().__init__(realm=realm, authfunc=authfunc)
         self.realm = realm
         self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
     def _get_response_from_code(self, http_code, fallback):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
             log.exception('Failed to fetch response class for code %s, using fallback: %s', http_code, fallback)
             return fallback
 
     def get_rc_realm(self):
         return safe_str(self.rc_realm)
 
     def build_authentication(self):
         header = [('WWW-Authenticate', f'Basic realm="{self.realm}"')]
 
         # NOTE: the initial_Call detection seems to be not working/not needed witg latest Mercurial
         # investigate if we still need it.
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
             custom_response_klass = self._get_response_from_code(self._rc_auth_http_code, fallback=HTTPUnauthorized)
             log.debug('Using custom response class: %s', custom_response_klass)
             return custom_response_klass(headers=header)
         return HTTPUnauthorized(headers=header)
 
     def authenticate(self, environ):
         authorization = paste.httpheaders.AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
         (auth_meth, auth_creds_b64) = authorization.split(' ', 1)
         if 'basic' != auth_meth.lower():
             return self.build_authentication()
 
         credentials = safe_str(base64.b64decode(auth_creds_b64.strip()))
         _parts = credentials.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name)
             if auth_data:
                 return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return sha1(
         config.get(b'beaker.session.secret', b'') + ascii_bytes(rhodecode.__version__)
     )[:8]
 
 
 def get_current_lang(request):
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id=None, is_api=None):
     """
     Attach variables into template context called `c`.
     """
     config = request.registry.settings
 
     rc_config = SettingsModel().get_all_settings(cache=True, from_request=False)
     context.rc_config = rc_config
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     context.rhodecode_edition_id = config.get('rhodecode.edition_id')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.show_revision_number = str2bool(
         rc_config.get('rhodecode_show_revision_number', True))
     context.visual.show_sha_length = safe_int(
         rc_config.get('rhodecode_show_sha_length', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
     context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.ssh_enabled = str2bool(
         config.get('ssh.generate_authorized_keyfile', 'false'))
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
         config.get('cut_off_limit_diff'), default=0)
     context.visual.cut_off_limit_file = safe_int(
         config.get('cut_off_limit_file'), default=0)
 
     context.license = AttributeDict({})
     context.license.hide_license_info = str2bool(
         config.get('license.hide_license_info', False))
 
     # AppEnlight
     context.appenlight_enabled = config.get('appenlight', False)
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     diffmode = {
         "unified": "unified",
         "sideside": "sideside"
     }.get(request.GET.get('diffmode'))
 
     if is_api is not None:
         is_api = hasattr(request, 'rpc_user')
     session_attrs = {
         # defaults
         "clone_url_format": "http",
         "diffmode": "sideside",
         "license_fingerprint": request.session.get('license_fingerprint')
     }
 
     if not is_api:
         # don't access pyramid session for API calls
         if diffmode and diffmode != request.session.get('rc_user_session_attr.diffmode'):
             request.session['rc_user_session_attr.diffmode'] = diffmode
 
         # session settings per user
 
         for k, v in list(request.session.items()):
             pref = 'rc_user_session_attr.'
             if k and k.startswith(pref):
                 k = k[len(pref):]
                 session_attrs[k] = v
 
     context.user_session_attrs = session_attrs
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'session_attrs': session_attrs,
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
     if is_api:
         csrf_token = None
     else:
         csrf_token = auth.get_csrf_token(session=request.session)
 
     context.csrf_token = csrf_token
     context.backends = list(rhodecode.BACKENDS.keys())
 
     unread_count = 0
     user_bookmark_list = []
     if user_id:
         unread_count = NotificationModel().get_unread_cnt_for_user(user_id)
         user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id)
     context.unread_notifications = unread_count
     context.bookmark_items = user_bookmark_list
 
     # web case
     if hasattr(request, 'user'):
         context.auth_user = request.user
         context.rhodecode_user = request.user
 
     # api case
     if hasattr(request, 'rpc_user'):
         context.auth_user = request.rpc_user
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
     request.set_call_context(context)
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
 
     # make sure that we update permissions each time we call controller
     _auth_token = (
             # ?auth_token=XXX
             request.GET.get('auth_token', '')
             # ?api_key=XXX !LEGACY
             or request.GET.get('api_key', '')
             # or headers....
             or request.headers.get('X-Rc-Auth-Token', '')
     )
     if not _auth_token and request.matchdict:
         url_auth_token = request.matchdict.get('_auth_token')
         _auth_token = url_auth_token
         if _auth_token:
             log.debug('Using URL extracted auth token `...%s`', _auth_token[-4:])
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         # in case someone changes a password for user it triggers session
         # flush and forces a re-login
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user, _auth_token
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
 
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
     config.add_route(name='main_page_repos_data', pattern='/_home_repos')
     config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
     config.add_route(name='hovercard_user',
                      pattern='/_hovercard/user/{user_id}')
 
     config.add_route(name='hovercard_user_group',
                      pattern='/_hovercard/user_group/{user_group_id}')
 
     config.add_route(name='hovercard_pull_request',
                      pattern='/_hovercard/pull_request/{pull_request_id}')
 
     config.add_route(name='hovercard_repo_commit',
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
 def bootstrap_config(request, registry_name='RcTestRegistry'):
-    from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
+    from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
     import pyramid.testing
     registry = pyramid.testing.Registry(registry_name)
 
     global_config = {'__file__': ''}
 
     config = pyramid.testing.setUp(registry=registry, request=request)
     sanitize_settings_and_apply_defaults(global_config, config.registry.settings)
 
     # allow pyramid lookup in testing
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.lib.rc_cache.archive_cache')
     add_events_routes(config)
 
     return config
 
 
 def bootstrap_request(**kwargs):
     """
     Returns a thin version of Request Object that is used in non-web context like testing/celery
     """
 
     import pyramid.testing
     from rhodecode.lib.request import ThinRequest as _ThinRequest
 
     class ThinRequest(_ThinRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
     class ThinSession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = ThinRequest(**kwargs)
     request.session = ThinSession()
 
     return request

rhodecode/lib/middleware/simplesvn.py

@@ -1,244 +1,243 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import base64
 import logging
 import urllib.request
 import urllib.parse
 import urllib.error
 import urllib.parse
 
 import requests
 from pyramid.httpexceptions import HTTPNotAcceptable
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.utils import get_path_info
 from rhodecode.lib.utils import is_valid_repo
 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
 from rhodecode.lib.type_utils import str2bool
 from rhodecode.lib.ext_json import json
-from rhodecode.lib.hooks_daemon import store_txn_id_data
-
+from rhodecode.lib.hook_daemon.base import store_txn_id_data
 
 log = logging.getLogger(__name__)
 
 
 class SimpleSvnApp(object):
     IGNORED_HEADERS = [
         'connection', 'keep-alive', 'content-encoding',
         'transfer-encoding', 'content-length']
     rc_extras = {}
 
     def __init__(self, config):
         self.config = config
         self.session = requests.Session()
 
     def __call__(self, environ, start_response):
         request_headers = self._get_request_headers(environ)
         data_io = environ['wsgi.input']
         req_method: str = environ['REQUEST_METHOD']
         has_content_length = 'CONTENT_LENGTH' in environ
 
         path_info = self._get_url(
             self.config.get('subversion_http_server_url', ''), get_path_info(environ))
         transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
         log.debug('Handling: %s method via `%s`', req_method, path_info)
 
         # stream control flag, based on request and content type...
         stream = False
 
         if req_method in ['MKCOL'] or has_content_length:
             data_processed = False
             # read chunk to check if we have txn-with-props
             initial_data: bytes = data_io.read(1024)
             if initial_data.startswith(b'(create-txn-with-props'):
                 data_io = initial_data + data_io.read()
                 # store on-the-fly our rc_extra using svn revision properties
                 # those can be read later on in hooks executed so we have a way
                 # to pass in the data into svn hooks
                 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
                 rc_data_len = str(len(rc_data))
                 # header defines data length, and serialized data
                 skel = b' rc-scm-extras %b %b' % (safe_bytes(rc_data_len), safe_bytes(rc_data))
                 data_io = data_io[:-2] + skel + b'))'
                 data_processed = True
 
             if not data_processed:
                 # NOTE(johbo): Avoid that we end up with sending the request in chunked
                 # transfer encoding (mainly on Gunicorn). If we know the content
                 # length, then we should transfer the payload in one request.
                 data_io = initial_data + data_io.read()
 
         if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
             # NOTE(marcink): when getting/uploading files, we want to STREAM content
             # back to the client/proxy instead of buffering it here...
             stream = True
 
         stream = stream
         log.debug('Calling SVN PROXY at `%s`, using method:%s. Stream: %s',
                   path_info, req_method, stream)
 
         call_kwargs = dict(
             data=data_io,
             headers=request_headers,
             stream=stream
         )
         if req_method in ['HEAD', 'DELETE']:
             del call_kwargs['data']
 
         try:
             response = self.session.request(
                 req_method, path_info, **call_kwargs)
         except requests.ConnectionError:
             log.exception('ConnectionError occurred for endpoint %s', path_info)
             raise
 
         if response.status_code not in [200, 401]:
             text = '\n{}'.format(safe_str(response.text)) if response.text else ''
             if response.status_code >= 500:
                 log.error('Got SVN response:%s with text:`%s`', response, text)
             else:
                 log.debug('Got SVN response:%s with text:`%s`', response, text)
         else:
             log.debug('got response code: %s', response.status_code)
 
         response_headers = self._get_response_headers(response.headers)
 
         if response.headers.get('SVN-Txn-name'):
             svn_tx_id = response.headers.get('SVN-Txn-name')
             txn_id = rc_cache.utils.compute_key_from_params(
                 self.config['repository'], svn_tx_id)
             port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
             store_txn_id_data(txn_id, {'port': port})
 
         start_response(f'{response.status_code} {response.reason}', response_headers)
         return response.iter_content(chunk_size=1024)
 
     def _get_url(self, svn_http_server, path):
         svn_http_server_url = (svn_http_server or '').rstrip('/')
         url_path = urllib.parse.urljoin(svn_http_server_url + '/', (path or '').lstrip('/'))
         url_path = urllib.parse.quote(url_path, safe="/:=~+!$,;'")
         return url_path
 
     def _get_request_headers(self, environ):
         headers = {}
         whitelist = {
             'Authorization': {}
         }
         for key in environ:
             if key in whitelist:
                 headers[key] = environ[key]
             elif not key.startswith('HTTP_'):
                 continue
             else:
                 new_key = key.split('_')
                 new_key = [k.capitalize() for k in new_key[1:]]
                 new_key = '-'.join(new_key)
                 headers[new_key] = environ[key]
 
         if 'CONTENT_TYPE' in environ:
             headers['Content-Type'] = environ['CONTENT_TYPE']
 
         if 'CONTENT_LENGTH' in environ:
             headers['Content-Length'] = environ['CONTENT_LENGTH']
 
         return headers
 
     def _get_response_headers(self, headers):
         headers = [
             (h, headers[h])
             for h in headers
             if h.lower() not in self.IGNORED_HEADERS
         ]
 
         return headers
 
 
 class DisabledSimpleSvnApp(object):
     def __init__(self, config):
         self.config = config
 
     def __call__(self, environ, start_response):
         reason = 'Cannot handle SVN call because: SVN HTTP Proxy is not enabled'
         log.warning(reason)
         return HTTPNotAcceptable(reason)(environ, start_response)
 
 
 class SimpleSvn(simplevcs.SimpleVCS):
 
     SCM = 'svn'
     READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
     DEFAULT_HTTP_SERVER = 'http://localhost:8090'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         path = get_path_info(environ).split('!')
         repo_name = path[0].strip('/')
 
         # SVN includes the whole path in it's requests, including
         # subdirectories inside the repo. Therefore we have to search for
         # the repo root directory.
         if not is_valid_repo(
                 repo_name, self.base_path, explicit_scm=self.SCM):
             current_path = ''
             for component in repo_name.split('/'):
                 current_path += component
                 if is_valid_repo(
                         current_path, self.base_path, explicit_scm=self.SCM):
                     return current_path
                 current_path += '/'
 
         return repo_name
 
     def _get_action(self, environ):
         return (
             'pull'
             if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
             else 'push')
 
     def _should_use_callback_daemon(self, extras, environ, action):
         # only MERGE command triggers hooks, so we don't want to start
         # hooks server too many times. POST however starts the svn transaction
         # so we also need to run the init of callback daemon of POST
         if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
             return True
         return False
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         if self._is_svn_enabled():
             return SimpleSvnApp(config)
         # we don't have http proxy enabled return dummy request handler
         return DisabledSimpleSvnApp(config)
 
     def _is_svn_enabled(self):
         conf = self.repo_vcs_config
         return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
 
     def _create_config(self, extras, repo_name, scheme='http'):
         conf = self.repo_vcs_config
         server_url = conf.get('vcs_svn_proxy', 'http_server_url')
         server_url = server_url or self.DEFAULT_HTTP_SERVER
 
         extras['subversion_http_server_url'] = server_url
         return extras

rhodecode/lib/middleware/simplevcs.py

@@ -1,701 +1,701 b''
 
 
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import io
 import logging
 import importlib
 from functools import wraps
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data: bytes):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
 
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = next(gen)
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'   # repo groups
         '(?P<target>{slug_pat})/'        # target repo
         'pull-request/(?P<pr_id>\\d+)/'  # pull request
         'repository$'                    # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
         realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
 
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_str('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         # Because PEP-3333-WSGI uses bytes-tunneled-in-latin-1 as PATH_INFO
         # and we use this data
         maybe_new_path = '/'.join(data)
         return safe_bytes(maybe_new_path).decode('latin1')
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access repo: `%s` '
                               'repository as anonymous user', self.acl_repo_name)
 
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container|headers based authentication')
 
                 # headers auth, by just reading special headers and bypass the auth with user/passwd
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
 
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got `%s` as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
 
                 # regular auth chain
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = str(user_id)
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
 
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
 
             stream = environ['wsgi.input']
 
             if isinstance(stream, io.BytesIO):
                 data: bytes = stream.getvalue()
             elif hasattr(stream, 'buf'):  # most likely gunicorn.http.body.Body
                 data: bytes = stream.buf.getvalue()
             else:
                 # fallback to the crudest way, copy the iterator
                 data = safe_bytes(stream.read())
                 environ['wsgi.input'] = io.BytesIO(data)
 
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         protocol = vcs_settings.HOOKS_PROTOCOL
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             protocol = 'local'
 
         return prepare_callback_daemon(
             extras, protocol=protocol,
             host=vcs_settings.HOOKS_HOST, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/lib/pyramid_utils.py

@@ -1,58 +1,48 b''
 
 
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import configparser
+
+from rhodecode.lib.config_utils import get_config
 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging  # pragma: no cover
 
-from rhodecode.lib.request import Request
-
-
-def get_config(ini_path, **kwargs):
-    parser = configparser.ConfigParser(**kwargs)
-    parser.read(ini_path)
-    return parser
-
-
-def get_app_config(ini_path):
-    from paste.deploy.loadwsgi import appconfig
-    return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
-
 
 def bootstrap(config_uri, options=None, env=None):
     from rhodecode.lib.utils2 import AttributeDict
+    from rhodecode.lib.request import Request
 
     if env:
         os.environ.update(env)
 
     config = get_config(config_uri)
     base_url = 'http://rhodecode.local'
     try:
         base_url = config.get('app:main', 'app.base_url')
     except (configparser.NoSectionError, configparser.NoOptionError):
         pass
 
     request = Request.blank('/', base_url=base_url)
     # fake inject a running user for bootstrap request !
     request.user = AttributeDict({'username': 'bootstrap-user',
                                   'user_id': 1,
                                   'ip_addr': '127.0.0.1'})
     return pyramid_bootstrap(config_uri, request=request, options=options)
 

rhodecode/lib/rc_commands/setup_rc.py

@@ -1,123 +1,124 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 import click
 import pyramid.paster
 
-from rhodecode.lib.pyramid_utils import bootstrap, get_app_config
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.config_utils import get_app_config
 from rhodecode.lib.db_manage import DbManage
 from rhodecode.lib.utils2 import get_encryption_key
 from rhodecode.model.db import Session
 
 
 log = logging.getLogger(__name__)
 
 
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
     '--force-yes/--force-no', default=None,
     help="Force yes/no to every question")
 @click.option(
     '--user',
     default=None,
     help='Initial super-admin username')
 @click.option(
     '--email',
     default=None,
     help='Initial super-admin email address.')
 @click.option(
     '--password',
     default=None,
     help='Initial super-admin password. Minimum 6 chars.')
 @click.option(
     '--api-key',
     help='Initial API key for the admin user')
 @click.option(
     '--repos',
     default=None,
     help='Absolute path to storage location. This is storage for all '
          'existing and future repositories, and repository groups.')
 @click.option(
     '--public-access/--no-public-access',
     default=None,
     help='Enable public access on this installation. '
          'Default is public access enabled.')
 @click.option(
     '--skip-existing-db',
     default=False,
     is_flag=True,
     help='Do not destroy and re-initialize the database if it already exist.')
 @click.option(
     '--apply-license-key',
     default=False,
     is_flag=True,
     help='Get the license key from a license file or ENV and apply during DB creation.')
 def main(ini_path, force_yes, user, email, password, api_key, repos,
          public_access, skip_existing_db, apply_license_key):
     return command(ini_path, force_yes, user, email, password, api_key,
                    repos, public_access, skip_existing_db, apply_license_key)
 
 
 def command(ini_path, force_yes, user, email, password, api_key, repos,
             public_access, skip_existing_db, apply_license_key):
     # mapping of old parameters to new CLI from click
     options = dict(
         username=user,
         email=email,
         password=password,
         api_key=api_key,
         repos_location=repos,
         force_ask=force_yes,
         public_access=public_access
     )
     pyramid.paster.setup_logging(ini_path)
 
     config = get_app_config(ini_path)
 
     db_uri = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
     dbmanage = DbManage(log_sql=True, dbconf=db_uri, root='.',
                         tests=False, cli_args=options, enc_key=enc_key)
     if skip_existing_db and dbmanage.db_exists():
         return
 
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     opts = dbmanage.config_prompt(None)
     dbmanage.create_settings(opts)
     dbmanage.create_default_user()
     dbmanage.create_admin_and_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     if apply_license_key:
         try:
             from rc_license.models import apply_trial_license_if_missing
             apply_trial_license_if_missing(force=True)
         except ImportError:
             pass
 
     Session().commit()
 
     with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
         msg = 'Successfully initialized database, schema and default data.'
         print()
         print('*' * len(msg))
         print(msg.upper())
         print('*' * len(msg))

rhodecode/lib/utils.py

@@ -1,859 +1,824 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 
 import decorator
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
-import urllib.parse
-import warnings
+
 from functools import wraps
 from os.path import join as jn
-from configparser import NoOptionError
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
 
 from mako import exceptions
 
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
-from rhodecode.lib.pyramid_utils import get_config
-from rhodecode.lib.vcs import CurlSession
-from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def adopt_for_celery(func):
     """
     Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
     for further usage as a celery tasks.
     """
     @wraps(func)
     def wrapper(extras):
         extras = AttributeDict(extras)
         # HooksResponse implements to_json method which must be used there.
         return func(extras).to_json()
     return wrapper
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     replacement_char = '-'
 
     slug = strip_tags(value)
     slug = convert_accented_entities(slug)
     slug = convert_misc_entities(slug)
 
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub(r'[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
 
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo_name'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     yield from _get_repos(rec_path)
 
     return _get_repos(path)
 
 
 def get_dirpaths(p: str) -> list:
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 "Ignoring path %s since it cannot be decoded into str.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if a given path is a repository group, False otherwise
 
     :param repo_group_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check; we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise OSError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
     for setting in ui_settings:
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in list(app_settings.items()):
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
     return rhodecode.CONFIG['default_base_path']
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in list(initial_repo_list.items()):
         group = map_groups(name)
         str_name = safe_str(name)
         db_repo = repo_model.get_by_repo_name(str_name)
 
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info(force=True)
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks(force=force_hooks_rebuild)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in list(initial_repo_list.keys()):
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    list(initial_repo_list.keys()) if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         def nested_sort(gr):
             return len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     try:
         import rc_testdata
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.lib.utils2 import get_encryption_key
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
 
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return sha256_safe(':'.join(uuid_list))
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)
-
-
-def call_service_api(ini_path, payload):
-    config = get_config(ini_path)
-    try:
-        host = config.get('app:main', 'app.service_api.host')
-    except NoOptionError:
-        raise ImproperlyConfiguredError(
-            "app.service_api.host is missing. "
-            "Please ensure that app.service_api.host and app.service_api.token are "
-            "defined inside of .ini configuration file."
-        )
-    try:
-        api_url = config.get('app:main', 'rhodecode.api.url')
-    except NoOptionError:
-        from rhodecode import api
-        log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
-        api_url = api.DEFAULT_URL
-
-    payload.update({
-        'id': 'service',
-        'auth_token': config.get('app:main', 'app.service_api.token')
-    })
-
-    response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
-
-    if response.status_code != 200:
-        raise Exception("Service API responded with error")
-
-    return json.loads(response.content)['result']

rhodecode/model/pull_request.py

@@ -1,2389 +1,2389 b''
 # Copyright (C) 2012-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 pull request model for RhodeCode
 """
 
 import logging
 import os
 
 import datetime
 import urllib.request
 import urllib.parse
 import urllib.error
 import collections
 
 import dataclasses as dataclasses
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.translation import lazy_ugettext
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
 from rhodecode.lib.hash_utils import md5_safe
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import AttributeDict, get_current_rhodecode_user
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason,
     TargetRefMissing, SourceRefMissing)
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError)
 from rhodecode.model import BaseModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     aliased, null, lazyload, and_, or_, select, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule, User)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
     EmailNotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 # Data structure to hold the response data when updating commits during a pull
 # request update.
 class UpdateResponse(object):
 
     def __init__(self, executed, reason, new, old, common_ancestor_id,
                  commit_changes, source_changed, target_changed):
 
         self.executed = executed
         self.reason = reason
         self.new = new
         self.old = old
         self.common_ancestor_id = common_ancestor_id
         self.changes = commit_changes
         self.source_changed = source_changed
         self.target_changed = target_changed
 
 
 def get_diff_info(
         source_repo, source_ref, target_repo, target_ref, get_authors=False,
         get_commit_authors=True):
     """
     Calculates detailed diff information for usage in preview of creation of a pull-request.
     This is also used for default reviewers logic
     """
 
     source_scm = source_repo.scm_instance()
     target_scm = target_repo.scm_instance()
 
     ancestor_id = target_scm.get_common_ancestor(target_ref, source_ref, source_scm)
     if not ancestor_id:
         raise ValueError(
             'cannot calculate diff info without a common ancestor. '
             'Make sure both repositories are related, and have a common forking commit.')
 
     # case here is that want a simple diff without incoming commits,
     # previewing what will be merged based only on commits in the source.
     log.debug('Using ancestor %s as source_ref instead of %s',
               ancestor_id, source_ref)
 
     # source of changes now is the common ancestor
     source_commit = source_scm.get_commit(commit_id=ancestor_id)
     # target commit becomes the source ref as it is the last commit
     # for diff generation this logic gives proper diff
     target_commit = source_scm.get_commit(commit_id=source_ref)
 
     vcs_diff = \
         source_scm.get_diff(commit1=source_commit, commit2=target_commit,
                             ignore_whitespace=False, context=3)
 
     diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
                                          diff_limit=0, file_limit=0, show_full_diff=True)
 
     _parsed = diff_processor.prepare()
 
     all_files = []
     all_files_changes = []
     changed_lines = {}
     stats = [0, 0]
     for f in _parsed:
         all_files.append(f['filename'])
         all_files_changes.append({
             'filename': f['filename'],
             'stats': f['stats']
         })
         stats[0] += f['stats']['added']
         stats[1] += f['stats']['deleted']
 
         changed_lines[f['filename']] = []
         if len(f['chunks']) < 2:
             continue
         # first line is "context" information
         for chunks in f['chunks'][1:]:
             for chunk in chunks['lines']:
                 if chunk['action'] not in ('del', 'mod'):
                     continue
                 changed_lines[f['filename']].append(chunk['old_lineno'])
 
     commit_authors = []
     user_counts = {}
     email_counts = {}
     author_counts = {}
     _commit_cache = {}
 
     commits = []
     if get_commit_authors:
         log.debug('Obtaining commit authors from set of commits')
         _compare_data = target_scm.compare(
             target_ref, source_ref, source_scm, merge=True,
             pre_load=["author", "date", "message"]
         )
 
         for commit in _compare_data:
             # NOTE(marcink): we serialize here, so we don't produce more vcsserver calls on data returned
             # at this function which is later called via JSON serialization
             serialized_commit = dict(
                 author=commit.author,
                 date=commit.date,
                 message=commit.message,
                 commit_id=commit.raw_id,
                 raw_id=commit.raw_id
             )
             commits.append(serialized_commit)
             user = User.get_from_cs_author(serialized_commit['author'])
             if user and user not in commit_authors:
                 commit_authors.append(user)
 
     # lines
     if get_authors:
         log.debug('Calculating authors of changed files')
         target_commit = source_repo.get_commit(ancestor_id)
 
         for fname, lines in changed_lines.items():
 
             try:
                 node = target_commit.get_node(fname, pre_load=["is_binary"])
             except Exception:
                 log.exception("Failed to load node with path %s", fname)
                 continue
 
             if not isinstance(node, FileNode):
                 continue
 
             # NOTE(marcink): for binary node we don't do annotation, just use last author
             if node.is_binary:
                 author = node.last_commit.author
                 email = node.last_commit.author_email
 
                 user = User.get_from_cs_author(author)
                 if user:
                     user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                 author_counts[author] = author_counts.get(author, 0) + 1
                 email_counts[email] = email_counts.get(email, 0) + 1
 
                 continue
 
             for annotation in node.annotate:
                 line_no, commit_id, get_commit_func, line_text = annotation
                 if line_no in lines:
                     if commit_id not in _commit_cache:
                         _commit_cache[commit_id] = get_commit_func()
                     commit = _commit_cache[commit_id]
                     author = commit.author
                     email = commit.author_email
                     user = User.get_from_cs_author(author)
                     if user:
                         user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                     author_counts[author] = author_counts.get(author, 0) + 1
                     email_counts[email] = email_counts.get(email, 0) + 1
 
     log.debug('Default reviewers processing finished')
 
     return {
         'commits': commits,
         'files': all_files_changes,
         'stats': stats,
         'ancestor': ancestor_id,
         # original authors of modified files
         'original_authors': {
             'users': user_counts,
             'authors': author_counts,
             'emails': email_counts,
         },
         'commit_authors': commit_authors
     }
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     DIFF_CONTEXT = diffs.DEFAULT_CONTEXT
 
     UPDATE_STATUS_MESSAGES = {
         UpdateFailureReason.NONE: lazy_ugettext(
             'Pull request update successful.'),
         UpdateFailureReason.UNKNOWN: lazy_ugettext(
             'Pull request update failed because of an unknown error.'),
         UpdateFailureReason.NO_CHANGE: lazy_ugettext(
             'No update needed because the source and target have not changed.'),
         UpdateFailureReason.WRONG_REF_TYPE: lazy_ugettext(
             'Pull request cannot be updated because the reference type is '
             'not supported for an update. Only Branch, Tag or Bookmark is allowed.'),
         UpdateFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be updated because the target '
             'reference is missing.'),
         UpdateFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
     REF_TYPES = ['bookmark', 'book', 'tag', 'branch']
     UPDATABLE_REF_TYPES = ['bookmark', 'book', 'branch']
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
             PullRequest, PullRequestVersion), pull_request)
 
     def _check_perms(self, perms, pull_request, user, api=False):
         if not api:
             return h.HasRepoPermissionAny(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
         else:
             return h.HasRepoPermissionAnyApi(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
 
     def check_user_read(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_merge(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'hg.admin',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_update(self, pull_request, user, api=False):
         owner = user.user_id == pull_request.user_id
         return self.check_user_merge(pull_request, user, api) or owner
 
     def check_user_delete(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         _perms = ('repository.admin',)
         return self._check_perms(_perms, pull_request, user) or owner
 
     def is_user_reviewer(self, pull_request, user):
         return user.user_id in [
             x.user_id for x in
             pull_request.get_pull_request_reviewers(PullRequestReviewers.ROLE_REVIEWER)
             if x.user
         ]
 
     def check_user_change_status(self, pull_request, user, api=False):
         return self.check_user_update(pull_request, user, api) \
                or self.is_user_reviewer(pull_request, user)
 
     def check_user_comment(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         return self.check_user_read(pull_request, user) or owner
 
     def get(self, pull_request):
         return self.__get_pull_request(pull_request)
 
     def _prepare_get_all_query(self, repo_name, search_q=None, source=False,
                                statuses=None, opened_by=None, order_by=None,
                                order_dir='desc', only_created=False):
         repo = None
         if repo_name:
             repo = self._get_repo(repo_name)
 
         q = PullRequest.query()
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         # source or target
         if repo and source:
             q = q.filter(PullRequest.source_repo == repo)
         elif repo:
             q = q.filter(PullRequest.target_repo == repo)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         # opened by filter
         if opened_by:
             q = q.filter(PullRequest.user_id.in_(opened_by))
 
         # only get those that are in "created" state
         if only_created:
             q = q.filter(PullRequest.pull_request_state == PullRequest.STATE_CREATED)
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'id': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_all(self, repo_name, search_q=None, source=False, statuses=None,
                   opened_by=None):
         """
         Count the number of pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by)
 
         return q.count()
 
     def get_all(self, repo_name, search_q=None, source=False, statuses=None,
                 opened_by=None, offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by, order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def count_awaiting_review(self, repo_name, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_review(
             repo_name, search_q=search_q, statuses=statuses)
 
         return len(pull_requests)
 
     def get_awaiting_review(self, repo_name, search_q=None, statuses=None,
                             offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         _filtered_pull_requests = []
         for pr in pull_requests:
             status = pr.calculated_review_status()
             if status in [ChangesetStatus.STATUS_NOT_REVIEWED,
                           ChangesetStatus.STATUS_UNDER_REVIEW]:
                 _filtered_pull_requests.append(pr)
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def _prepare_awaiting_my_review_review_query(
             self, repo_name, user_id, search_q=None, statuses=None,
             order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
         repo_alias = aliased(Repository)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .join(repo_alias,
                   repo_alias.repo_id == pull_request_alias.target_repo_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(repo_alias.repo_name == repo_name) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses)
         return q.count()
 
     def get_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None,
                                offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
 
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_im_participating_query(self, user_id=None, statuses=None, query='',
                                         order_by=None, order_dir='desc'):
         """
         return a query of pull-requests user is an creator, or he's added as a reviewer
         """
         q = PullRequest.query()
         if user_id:
 
             base_query = select(PullRequestReviewers)\
                 .where(PullRequestReviewers.user_id == user_id)\
                 .with_only_columns(PullRequestReviewers.pull_request_id)
 
             user_filter = or_(
                 PullRequest.user_id == user_id,
                 PullRequest.pull_request_id.in_(base_query)
             )
             q = PullRequest.query().filter(user_filter)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in(self, user_id=None, statuses=None, query=''):
         q = self._prepare_im_participating_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in(
             self, user_id=None, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that i'm participating in as a reviewer, or i have opened
         """
 
         q = self._prepare_im_participating_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_participating_in_for_review_query(
             self, user_id, statuses=None, query='', order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in_for_review(self, user_id, statuses=None, query=''):
         q = self._prepare_participating_in_for_review_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in_for_review(
             self, user_id, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that needs user approval or rejection
         """
 
         q = self._prepare_participating_in_for_review_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def get_versions(self, pull_request):
         """
         returns version of pull request sorted by ID descending
         """
         return PullRequestVersion.query()\
             .filter(PullRequestVersion.pull_request == pull_request)\
             .order_by(PullRequestVersion.pull_request_version_id.asc())\
             .all()
 
     def get_pr_version(self, pull_request_id, version=None):
         at_version = None
 
         if version and version == 'latest':
             pull_request_ver = PullRequest.get(pull_request_id)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_obj
             at_version = 'latest'
         elif version:
             pull_request_ver = PullRequestVersion.get_or_404(version)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_ver.pull_request
             at_version = pull_request_ver.pull_request_version_id
         else:
             _org_pull_request_obj = pull_request_obj = PullRequest.get_or_404(
                 pull_request_id)
 
         pull_request_display_obj = PullRequest.get_pr_display_object(
             pull_request_obj, _org_pull_request_obj)
 
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
     def pr_commits_versions(self, versions):
         """
         Maps the pull-request commits into all known PR versions. This way we can obtain
         each pr version the commit was introduced in.
         """
         commit_versions = collections.defaultdict(list)
         num_versions = [x.pull_request_version_id for x in versions]
         for ver in versions:
             for commit_id in ver.revisions:
                 ver_idx = ChangesetComment.get_index_from_version(
                     ver.pull_request_version_id, num_versions=num_versions)
                 commit_versions[commit_id].append(ver_idx)
         return commit_versions
 
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, observers, title, description=None,
                common_ancestor_id=None,
                description_renderer=None,
                reviewer_data=None, translator=None, auth_user=None):
         translator = translator or get_current_request().translate
 
         created_by_user = self._get_user(created_by)
         auth_user = auth_user or created_by_user.AuthUser()
         source_repo = self._get_repo(source_repo)
         target_repo = self._get_repo(target_repo)
 
         pull_request = PullRequest()
         pull_request.source_repo = source_repo
         pull_request.source_ref = source_ref
         pull_request.target_repo = target_repo
         pull_request.target_ref = target_ref
         pull_request.revisions = revisions
         pull_request.title = title
         pull_request.description = description
         pull_request.description_renderer = description_renderer
         pull_request.author = created_by_user
         pull_request.reviewer_data = reviewer_data
         pull_request.pull_request_state = pull_request.STATE_CREATING
         pull_request.common_ancestor_id = common_ancestor_id
 
         Session().add(pull_request)
         Session().flush()
 
         reviewer_ids = set()
         # members / reviewers
         for reviewer_object in reviewers:
             user_id, reasons, mandatory, role, rules = reviewer_object
             user = self._get_user(user_id)
 
             # skip duplicates
             if user.user_id in reviewer_ids:
                 continue
 
             reviewer_ids.add(user.user_id)
 
             reviewer = PullRequestReviewers()
             reviewer.user = user
             reviewer.pull_request = pull_request
             reviewer.reasons = reasons
             reviewer.mandatory = mandatory
             reviewer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 review_group = rule.user_group_vote_rule(user_id)
                 # we check if this particular reviewer is member of a voting group
                 if review_group:
                     # NOTE(marcink):
                     # can be that user is member of more but we pick the first same,
                     # same as default reviewers algo
                     review_group = review_group[0]
 
                     rule_data = {
                         'rule_name':
                             rule.review_rule_name,
                         'rule_user_group_entry_id':
                             review_group.repo_review_rule_users_group_id,
                         'rule_user_group_name':
                             review_group.users_group.users_group_name,
                         'rule_user_group_members':
                             [x.user.username for x in review_group.users_group.members],
                         'rule_user_group_members_id':
                             [x.user.user_id for x in review_group.users_group.members],
                     }
                     # e.g {'vote_rule': -1, 'mandatory': True}
                     rule_data.update(review_group.rule_data())
 
                     reviewer.rule_data = rule_data
 
             Session().add(reviewer)
             Session().flush()
 
         for observer_object in observers:
             user_id, reasons, mandatory, role, rules = observer_object
             user = self._get_user(user_id)
 
             # skip duplicates from reviewers
             if user.user_id in reviewer_ids:
                 continue
 
             #reviewer_ids.add(user.user_id)
 
             observer = PullRequestReviewers()
             observer.user = user
             observer.pull_request = pull_request
             observer.reasons = reasons
             observer.mandatory = mandatory
             observer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 # TODO(marcink): do we need this for observers ??
                 pass
 
             Session().add(observer)
             Session().flush()
 
         # Set approval status to "Under Review" for all commits which are
         # part of this pull request.
         ChangesetStatusModel().set_status(
             repo=target_repo,
             status=ChangesetStatus.STATUS_UNDER_REVIEW,
             user=created_by_user,
             pull_request=pull_request
         )
         # we commit early at this point. This has to do with a fact
         # that before queries do some row-locking. And because of that
         # we need to commit and finish transaction before below validate call
         # that for large repos could be long resulting in long row locks
         Session().commit()
 
         # prepare workspace, and run initial merge simulation. Set state during that
         # operation
         pull_request = PullRequest.get(pull_request.pull_request_id)
 
         # set as merging, for merge simulation, and if finished to created so we mark
         # simulation is working fine
         with pull_request.set_state(PullRequest.STATE_MERGING,
                                     final_state=PullRequest.STATE_CREATED) as state_obj:
             MergeCheck.validate(
                 pull_request, auth_user=auth_user, translator=translator)
 
         self.notify_reviewers(pull_request, reviewer_ids, created_by_user)
         self.trigger_pull_request_hook(pull_request, created_by_user, 'create')
 
         creation_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.create', {'data': creation_data},
             auth_user, pull_request)
 
         return pull_request
 
     def trigger_pull_request_hook(self, pull_request, user, action, data=None):
         pull_request = self.__get_pull_request(pull_request)
         target_scm = pull_request.target_repo.scm_instance()
         if action == 'create':
             trigger_hook = hooks_utils.trigger_create_pull_request_hook
         elif action == 'merge':
             trigger_hook = hooks_utils.trigger_merge_pull_request_hook
         elif action == 'close':
             trigger_hook = hooks_utils.trigger_close_pull_request_hook
         elif action == 'review_status_change':
             trigger_hook = hooks_utils.trigger_review_pull_request_hook
         elif action == 'update':
             trigger_hook = hooks_utils.trigger_update_pull_request_hook
         elif action == 'comment':
             trigger_hook = hooks_utils.trigger_comment_pull_request_hook
         elif action == 'comment_edit':
             trigger_hook = hooks_utils.trigger_comment_pull_request_edit_hook
         else:
             return
 
         log.debug('Handling pull_request %s trigger_pull_request_hook with action %s and hook: %s',
                   pull_request, action, trigger_hook)
         trigger_hook(
             username=user.username,
             repo_name=pull_request.target_repo.repo_name,
             repo_type=target_scm.alias,
             pull_request=pull_request,
             data=data)
 
     def _get_commit_ids(self, pull_request):
         """
         Return the commit ids of the merged pull request.
 
         This method is not dealing correctly yet with the lack of autoupdates
         nor with the implicit target updates.
         For example: if a commit in the source repo is already in the target it
         will be reported anyways.
         """
         merge_rev = pull_request.merge_rev
         if merge_rev is None:
             raise ValueError('This pull request was not merged yet')
 
         commit_ids = list(pull_request.revisions)
         if merge_rev not in commit_ids:
             commit_ids.append(merge_rev)
 
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
         repo_type = pull_request.source_repo.repo_type
         log.debug("Merging pull request %s", pull_request)
 
         extras['user_agent'] = '{}/internal-merge'.format(repo_type)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug("Merge was successful, updating the pull request comments.")
             self._comment_and_close_pr(pull_request, user, merge_state)
 
             self._log_audit_action(
                 'repo.pull_request.merge',
                 {'merge_state': merge_state.__dict__},
                 user, pull_request)
 
         else:
             log.warning("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
 
         message = safe_str(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
             pr_id=pull_request.pull_request_id,
             pr_title=pull_request.title,
             pr_desc=pull_request.description,
             source_repo=source_vcs.name,
             source_ref_name=pull_request.source_ref_parts.name,
             target_repo=target_vcs.name,
             target_ref_name=pull_request.target_ref_parts.name,
         )
 
         workspace_id = self._workspace_id(pull_request)
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         user_name = self._user_name_for_merging(pull_request, user)
 
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override
             # for a single call.
             target_vcs.config.set(
                 'rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
             merge_state = target_vcs.merge(
                 repo_id, workspace_id, target_ref, source_vcs,
                 pull_request.source_ref_parts,
                 user_name=user_name, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
 
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
         pull_request.merge_rev = merge_state.merge_ref.commit_id
         pull_request.updated_on = datetime.datetime.now()
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
             text=safe_str(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             closing_pr=True
         )
 
         Session().add(pull_request)
         Session().flush()
         # TODO: paris: replace invalidation with less radical solution
         ScmModel().mark_for_invalidation(
             pull_request.target_repo.repo_name)
         self.trigger_pull_request_hook(pull_request, user, 'merge')
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
         return source_ref_type in self.REF_TYPES
 
     def get_flow_commits(self, pull_request):
 
         # source repo
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_id = pull_request.source_ref_parts.commit_id
         source_repo = pull_request.source_repo.scm_instance()
 
         try:
             if source_ref_type in self.REF_TYPES:
                 source_commit = source_repo.get_commit(
                     source_ref_name, reference_obj=pull_request.source_ref_parts)
             else:
                 source_commit = source_repo.get_commit(source_ref_id)
         except CommitDoesNotExistError:
             raise SourceRefMissing()
 
         # target repo
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_id = pull_request.target_ref_parts.commit_id
         target_repo = pull_request.target_repo.scm_instance()
 
         try:
             if target_ref_type in self.REF_TYPES:
                 target_commit = target_repo.get_commit(
                     target_ref_name, reference_obj=pull_request.target_ref_parts)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
         except CommitDoesNotExistError:
             raise TargetRefMissing()
 
         return source_commit, target_commit
 
     def update_commits(self, pull_request, updating_user):
         """
         Get the updated list of commits for the pull request
         and return the new pull request version and the list
         of commits processed by this update action
 
         updating_user is the user_object who triggered the update
         """
         pull_request = self.__get_pull_request(pull_request)
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_id = pull_request.source_ref_parts.commit_id
 
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         if not self.has_valid_update_type(pull_request):
             log.debug("Skipping update of pull request %s due to ref type: %s",
                       pull_request, source_ref_type)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.WRONG_REF_TYPE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         try:
             source_commit, target_commit = self.get_flow_commits(pull_request)
         except SourceRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_SOURCE_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
         except TargetRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         source_changed = source_ref_id != source_commit.raw_id
         target_changed = target_ref_id != target_commit.raw_id
 
         if not (source_changed or target_changed):
             log.debug("Nothing changed in pull request %s", pull_request)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.NO_CHANGE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=target_changed, target_changed=source_changed)
 
         change_in_found = 'target repo' if target_changed else 'source repo'
         log.debug('Updating pull request because of change in %s detected',
                   change_in_found)
 
         # Finally there is a need for an update, in case of source change
         # we create a new version, else just an update
         if source_changed:
             pull_request_version = self._create_version_from_snapshot(pull_request)
             self._link_comments_to_version(pull_request_version)
         else:
             try:
                 ver = pull_request.versions[-1]
             except IndexError:
                 ver = None
 
             pull_request.pull_request_version_id = \
                 ver.pull_request_version_id if ver else None
             pull_request_version = pull_request
 
         source_repo = pull_request.source_repo.scm_instance()
         target_repo = pull_request.target_repo.scm_instance()
 
         # re-compute commit ids
         old_commit_ids = pull_request.revisions
         pre_load = ["author", "date", "message", "branch"]
         commit_ranges = target_repo.compare(
             target_commit.raw_id, source_commit.raw_id, source_repo, merge=True,
             pre_load=pre_load)
 
         target_ref = target_commit.raw_id
         source_ref = source_commit.raw_id
         ancestor_commit_id = target_repo.get_common_ancestor(
             target_ref, source_ref, source_repo)
 
         if not ancestor_commit_id:
             raise ValueError(
                 'cannot calculate diff info without a common ancestor. '
                 'Make sure both repositories are related, and have a common forking commit.')
 
         pull_request.common_ancestor_id = ancestor_commit_id
 
         pull_request.source_ref = f'{source_ref_type}:{source_ref_name}:{source_commit.raw_id}'
         pull_request.target_ref = f'{target_ref_type}:{target_ref_name}:{ancestor_commit_id}'
 
         pull_request.revisions = [
             commit.raw_id for commit in reversed(commit_ranges)]
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         new_commit_ids = pull_request.revisions
 
         old_diff_data, new_diff_data = self._generate_update_diffs(
             pull_request, pull_request_version)
 
         # calculate commit and file changes
         commit_changes = self._calculate_commit_id_changes(
             old_commit_ids, new_commit_ids)
         file_changes = self._calculate_file_changes(
             old_diff_data, new_diff_data)
 
         # set comments as outdated if DIFFS changed
         CommentsModel().outdate_comments(
             pull_request, old_diff_data=old_diff_data,
             new_diff_data=new_diff_data)
 
         valid_commit_changes = (commit_changes.added or commit_changes.removed)
         file_node_changes = (
             file_changes.added or file_changes.modified or file_changes.removed)
         pr_has_changes = valid_commit_changes or file_node_changes
 
         # Add an automatic comment to the pull request, in case
         # anything has changed
         if pr_has_changes:
             update_comment = CommentsModel().create(
                 text=self._render_update_message(ancestor_commit_id, commit_changes, file_changes),
                 repo=pull_request.target_repo,
                 user=pull_request.author,
                 pull_request=pull_request,
                 send_email=False, renderer=DEFAULT_COMMENTS_RENDERER)
 
             # Update status to "Under Review" for added commits
             for commit_id in commit_changes.added:
                 ChangesetStatusModel().set_status(
                     repo=pull_request.source_repo,
                     status=ChangesetStatus.STATUS_UNDER_REVIEW,
                     comment=update_comment,
                     user=pull_request.author,
                     pull_request=pull_request,
                     revision=commit_id)
 
         # initial commit
         Session().commit()
 
         if pr_has_changes:
             # send update email to users
             try:
                 self.notify_users(pull_request=pull_request, updating_user=updating_user,
                                   ancestor_commit_id=ancestor_commit_id,
                                   commit_changes=commit_changes,
                                   file_changes=file_changes)
                 Session().commit()
             except Exception:
                 log.exception('Failed to send email notification to users')
                 Session().rollback()
 
         log.debug(
             'Updated pull request %s, added_ids: %s, common_ids: %s, '
             'removed_ids: %s', pull_request.pull_request_id,
             commit_changes.added, commit_changes.common, commit_changes.removed)
         log.debug(
             'Updated pull request with the following file changes: %s',
             file_changes)
 
         log.info(
             "Updated pull request %s from commit %s to commit %s, "
             "stored new version %s of this pull request.",
             pull_request.pull_request_id, source_ref_id,
             pull_request.source_ref_parts.commit_id,
             pull_request_version.pull_request_version_id)
 
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'update')
 
         return UpdateResponse(
             executed=True, reason=UpdateFailureReason.NONE,
             old=pull_request, new=pull_request_version,
             common_ancestor_id=ancestor_commit_id, commit_changes=commit_changes,
             source_changed=source_changed, target_changed=target_changed)
 
     def _create_version_from_snapshot(self, pull_request):
         version = PullRequestVersion()
         version.title = pull_request.title
         version.description = pull_request.description
         version.status = pull_request.status
         version.pull_request_state = pull_request.pull_request_state
         version.created_on = datetime.datetime.now()
         version.updated_on = pull_request.updated_on
         version.user_id = pull_request.user_id
         version.source_repo = pull_request.source_repo
         version.source_ref = pull_request.source_ref
         version.target_repo = pull_request.target_repo
         version.target_ref = pull_request.target_ref
 
         version._last_merge_source_rev = pull_request._last_merge_source_rev
         version._last_merge_target_rev = pull_request._last_merge_target_rev
         version.last_merge_status = pull_request.last_merge_status
         version.last_merge_metadata = pull_request.last_merge_metadata
         version.shadow_merge_ref = pull_request.shadow_merge_ref
         version.merge_rev = pull_request.merge_rev
         version.reviewer_data = pull_request.reviewer_data
 
         version.revisions = pull_request.revisions
         version.common_ancestor_id = pull_request.common_ancestor_id
         version.pull_request = pull_request
         Session().add(version)
         Session().flush()
 
         return version
 
     def _generate_update_diffs(self, pull_request, pull_request_version):
 
         diff_context = (
             self.DIFF_CONTEXT +
             CommentsModel.needed_extra_diff_context())
         hide_whitespace_changes = False
         source_repo = pull_request_version.source_repo
         source_ref_id = pull_request_version.source_ref_parts.commit_id
         target_ref_id = pull_request_version.target_ref_parts.commit_id
         old_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         new_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         # NOTE: this was using diff_format='gitdiff'
         old_diff_data = diffs.DiffProcessor(old_diff, diff_format='newdiff')
         old_diff_data.prepare()
         new_diff_data = diffs.DiffProcessor(new_diff, diff_format='newdiff')
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
 
     def _link_comments_to_version(self, pull_request_version):
         """
         Link all unlinked comments of this pull request to the given version.
 
         :param pull_request_version: The `PullRequestVersion` to which
             the comments shall be linked.
 
         """
         pull_request = pull_request_version.pull_request
         comments = ChangesetComment.query()\
             .filter(
                 # TODO: johbo: Should we query for the repo at all here?
                 # Pending decision on how comments of PRs are to be related
                 # to either the source repo, the target repo or no repo at all.
                 ChangesetComment.repo_id == pull_request.target_repo.repo_id,
                 ChangesetComment.pull_request == pull_request,
                 ChangesetComment.pull_request_version == null())\
             .order_by(ChangesetComment.comment_id.asc())
 
         # TODO: johbo: Find out why this breaks if it is done in a bulk
         # operation.
         for comment in comments:
             comment.pull_request_version_id = (
                 pull_request_version.pull_request_version_id)
             Session().add(comment)
 
     def _calculate_commit_id_changes(self, old_ids, new_ids):
         added = [x for x in new_ids if x not in old_ids]
         common = [x for x in new_ids if x in old_ids]
         removed = [x for x in old_ids if x not in new_ids]
         total = new_ids
         return ChangeTuple(added, common, removed, total)
 
     def _calculate_file_changes(self, old_diff_data, new_diff_data):
 
         old_files = OrderedDict()
         for diff_data in old_diff_data.parsed_diff:
             old_files[diff_data['filename']] = md5_safe(diff_data['raw_diff'])
 
         added_files = []
         modified_files = []
         removed_files = []
         for diff_data in new_diff_data.parsed_diff:
             new_filename = diff_data['filename']
             new_hash = md5_safe(diff_data['raw_diff'])
 
             old_hash = old_files.get(new_filename)
             if not old_hash:
                 # file is not present in old diff, we have to figure out from parsed diff
                 # operation ADD/REMOVE
                 operations_dict = diff_data['stats']['ops']
                 if diffs.DEL_FILENODE in operations_dict:
                     removed_files.append(new_filename)
                 else:
                     added_files.append(new_filename)
             else:
                 if new_hash != old_hash:
                     modified_files.append(new_filename)
                 # now remove a file from old, since we have seen it already
                 del old_files[new_filename]
 
         # removed files is when there are present in old, but not in NEW,
         # since we remove old files that are present in new diff, left-overs
         # if any should be the removed files
         removed_files.extend(old_files.keys())
 
         return FileChangeTuple(added_files, modified_files, removed_files)
 
     def _render_update_message(self, ancestor_commit_id, changes, file_changes):
         """
         render the message using DEFAULT_COMMENTS_RENDERER (RST renderer),
         so it's always looking the same disregarding on which default
         renderer system is using.
 
         :param ancestor_commit_id: ancestor raw_id
         :param changes: changes named tuple
         :param file_changes: file changes named tuple
 
         """
         new_status = ChangesetStatus.get_status_lbl(
             ChangesetStatus.STATUS_UNDER_REVIEW)
 
         changed_files = (
             file_changes.added + file_changes.modified + file_changes.removed)
 
         params = {
             'under_review_label': new_status,
             'added_commits': changes.added,
             'removed_commits': changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'ancestor_commit_id': ancestor_commit_id
         }
         renderer = RstTemplateRenderer()
         return renderer.render('pull_request_update.mako', **params)
 
     def edit(self, pull_request, title, description, description_renderer, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
         if title:
             pull_request.title = title
         pull_request.description = description
         pull_request.updated_on = datetime.datetime.now()
         pull_request.description_renderer = description_renderer
         Session().add(pull_request)
         self._log_audit_action(
             'repo.pull_request.edit', {'old_data': old_data},
             user, pull_request)
 
     def update_reviewers(self, pull_request, reviewer_data, user):
         """
         Update the reviewers in the pull request
 
         :param pull_request: the pr to update
         :param reviewer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
 
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         reviewers = {}
         for user_id, reasons, mandatory, role, rules in reviewer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             reviewers[user_id] = {
                 'reasons': reasons, 'mandatory': mandatory, 'role': role}
 
         reviewers_ids = set(reviewers.keys())
         current_reviewers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_REVIEWER)
 
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         ids_to_add = reviewers_ids.difference(current_reviewers_ids)
         ids_to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers", ids_to_add)
         log.debug("Removing %s reviewers", ids_to_remove)
         changed = False
         added_audit_reviewers = []
         removed_audit_reviewers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             reviewer = PullRequestReviewers()
             reviewer.user = _usr
             reviewer.pull_request = pull_request
             reviewer.reasons = reviewers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # reviewer.mandatory = reviewers[uid]['reasons']
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             reviewer.role = PullRequestReviewers.ROLE_REVIEWER
             Session().add(reviewer)
             added_audit_reviewers.append(reviewer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             reviewers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_REVIEWER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in reviewers:
                 added_audit_reviewers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_reviewers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def update_observers(self, pull_request, observer_data, user):
         """
         Update the observers in the pull request
 
         :param pull_request: the pr to update
         :param observer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         observers = {}
         for user_id, reasons, mandatory, role, rules in observer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             observers[user_id] = {
                 'reasons': reasons, 'observers': mandatory, 'role': role}
 
         observers_ids = set(observers.keys())
         current_observers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_OBSERVER)
 
         current_observers_ids = set([x.user.user_id for x in current_observers])
 
         ids_to_add = observers_ids.difference(current_observers_ids)
         ids_to_remove = current_observers_ids.difference(observers_ids)
 
         log.debug("Adding %s observer", ids_to_add)
         log.debug("Removing %s observer", ids_to_remove)
         changed = False
         added_audit_observers = []
         removed_audit_observers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             observer = PullRequestReviewers()
             observer.user = _usr
             observer.pull_request = pull_request
             observer.reasons = observers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # observer.mandatory = observer[uid]['reasons']
 
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             observer.role = PullRequestReviewers.ROLE_OBSERVER
             Session().add(observer)
             added_audit_observers.append(observer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             observers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_OBSERVER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in observers:
                 added_audit_observers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_observers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def get_url(self, pull_request, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if permalink:
             return request.route_url(
                 'pull_requests_global',
                 pull_request_id=pull_request.pull_request_id,)
         else:
             return request.route_url('pullrequest_show',
                 repo_name=safe_str(pull_request.target_repo.repo_name),
                 pull_request_id=pull_request.pull_request_id,)
 
     def get_shadow_clone_url(self, pull_request, request=None):
         """
         Returns qualified url pointing to the shadow repository. If this pull
         request is closed there is no shadow repository and ``None`` will be
         returned.
         """
         if pull_request.is_closed():
             return None
         else:
             pr_url = urllib.parse.unquote(self.get_url(pull_request, request=request))
             return safe_str('{pr_url}/repository'.format(pr_url=pr_url))
 
     def _notify_reviewers(self, pull_request, user_ids, role, user):
         # notification to reviewers/observers
         if not user_ids:
             return
 
         log.debug('Notify following %s users about pull-request %s', role, user_ids)
 
         pull_request_obj = pull_request
         # get the current participants of this pull request
         recipients = user_ids
         notification_type = EmailNotificationModel.TYPE_PULL_REQUEST
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         # pull request specifics
         pull_request_commits = [
             (x.raw_id, x.message)
             for x in map(pr_source_repo.get_commit, pull_request.revisions)]
 
         current_rhodecode_user = user
         kwargs = {
             'user': current_rhodecode_user,
             'pull_request_author': pull_request.author,
             'pull_request': pull_request_obj,
             'pull_request_commits': pull_request_commits,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
             'thread_ids': [pr_url],
             'user_role': role
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=current_rhodecode_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=notification_type,
             recipients=recipients,
             email_kwargs=kwargs,
         )
 
     def notify_reviewers(self, pull_request, reviewers_ids, user):
         return self._notify_reviewers(pull_request, reviewers_ids,
                                       PullRequestReviewers.ROLE_REVIEWER, user)
 
     def notify_observers(self, pull_request, observers_ids, user):
         return self._notify_reviewers(pull_request, observers_ids,
                                       PullRequestReviewers.ROLE_OBSERVER, user)
 
     def notify_users(self, pull_request, updating_user, ancestor_commit_id,
                      commit_changes, file_changes):
 
         updating_user_id = updating_user.user_id
         reviewers = set([x.user.user_id for x in pull_request.get_pull_request_reviewers()])
         # NOTE(marcink): send notification to all other users except to
         # person who updated the PR
         recipients = reviewers.difference(set([updating_user_id]))
 
         log.debug('Notify following recipients about pull-request update %s', recipients)
 
         pull_request_obj = pull_request
 
         # send email about the update
         changed_files = (
                 file_changes.added + file_changes.modified + file_changes.removed)
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'updating_user': updating_user,
 
             'pull_request': pull_request_obj,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
 
             'ancestor_commit_id': ancestor_commit_id,
             'added_commits': commit_changes.added,
             'removed_commits': commit_changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'thread_ids': [pr_url],
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=updating_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=EmailNotificationModel.TYPE_PULL_REQUEST_UPDATE,
             recipients=recipients,
             email_kwargs=email_kwargs,
         )
 
     def delete(self, pull_request, user=None):
         if not user:
             user = getattr(get_current_rhodecode_user(), 'username', None)
 
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         self._cleanup_merge_workspace(pull_request)
         self._log_audit_action(
             'repo.pull_request.delete', {'old_data': old_data},
             user, pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         self._cleanup_merge_workspace(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'close')
 
         pr_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
             self, pull_request, user, repo, message=None, auth_user=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
         if pull_request_review_status == ChangesetStatus.STATUS_APPROVED:
             # approved only if we have voting consent
             status = ChangesetStatus.STATUS_APPROVED
         else:
             status = ChangesetStatus.STATUS_REJECTED
         status_lbl = ChangesetStatus.get_status_lbl(status)
 
         default_message = (
             'Closing with status change {transition_icon} {status}.'
         ).format(transition_icon='>', status=status_lbl)
         text = message or default_message
 
         # create a comment, and link it to new status
         comment = CommentsModel().create(
             text=text,
             repo=repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
             closing_pr=True,
             auth_user=auth_user,
         )
 
         # calculate old status before we change it
         old_calculated_status = pull_request.calculated_review_status()
         ChangesetStatusModel().set_status(
             repo.repo_id,
             status,
             user.user_id,
             comment=comment,
             pull_request=pull_request.pull_request_id
         )
 
         Session().flush()
 
         self.trigger_pull_request_hook(pull_request, user, 'comment',
                                        data={'comment': comment})
 
         # we now calculate the status of pull request again, and based on that
         # calculation trigger status change. This might happen in cases
         # that non-reviewer admin closes a pr, which means his vote doesn't
         # change the status, while if he's a reviewer this might change it.
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             self.trigger_pull_request_hook(pull_request, user, 'review_status_change',
                                            data={'status': calculated_status})
 
         # finally close the PR
         PullRequestModel().close_pull_request(pull_request.pull_request_id, user)
 
         return comment, status
 
     def merge_status(self, pull_request, translator=None, force_shadow_repo_refresh=False):
         _ = translator or get_current_request().translate
 
         if not self._is_merge_enabled(pull_request):
             return None, False, _('Server-side pull request merging is disabled.')
 
         if pull_request.is_closed():
             return None, False, _('This pull request is closed.')
 
         merge_possible, msg = self._check_repo_requirements(
             target=pull_request.target_repo, source=pull_request.source_repo,
             translator=_)
         if not merge_possible:
             return None, merge_possible, msg
 
         try:
             merge_response = self._try_merge(
                 pull_request, force_shadow_repo_refresh=force_shadow_repo_refresh)
             log.debug("Merge response: %s", merge_response)
             return merge_response, merge_response.possible, merge_response.merge_status_message
         except NotImplementedError:
             return None, False, _('Pull request merging is not supported.')
 
     def _check_repo_requirements(self, target, source, translator):
         """
         Check if `target` and `source` have compatible requirements.
 
         Currently this is just checking for largefiles.
         """
         _ = translator
         target_has_largefiles = self._has_largefiles(target)
         source_has_largefiles = self._has_largefiles(source)
         merge_possible = True
         message = u''
 
         if target_has_largefiles != source_has_largefiles:
             merge_possible = False
             if source_has_largefiles:
                 message = _(
                     'Target repository large files support is disabled.')
             else:
                 message = _(
                     'Source repository large files support is disabled.')
 
         return merge_possible, message
 
     def _has_largefiles(self, repo):
         largefiles_ui = VcsSettingsModel(repo=repo).get_ui_settings(
             'extensions', 'largefiles')
         return largefiles_ui and largefiles_ui[0].active
 
     def _try_merge(self, pull_request, force_shadow_repo_refresh=False):
         """
         Try to merge the pull request and return the merge status.
         """
         log.debug(
             "Trying out if the pull request %s can be merged. Force_refresh=%s",
             pull_request.pull_request_id, force_shadow_repo_refresh)
         target_vcs = pull_request.target_repo.scm_instance()
         # Refresh the target reference.
         try:
             target_ref = self._refresh_reference(
                 pull_request.target_ref_parts, target_vcs)
         except CommitDoesNotExistError:
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF,
                 metadata={'target_ref': pull_request.target_ref_parts})
             return merge_state
 
         target_locked = pull_request.target_repo.locked
         if target_locked and target_locked[0]:
             locked_by = 'user:{}'.format(target_locked[0])
             log.debug("The target repository is locked by %s.", locked_by)
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_LOCKED,
                 metadata={'locked_by': locked_by})
         elif force_shadow_repo_refresh or self._needs_merge_state_refresh(
                 pull_request, target_ref):
             log.debug("Refreshing the merge status of the repository.")
             merge_state = self._refresh_merge_state(
                 pull_request, target_vcs, target_ref)
         else:
             possible = pull_request.last_merge_status == MergeFailureReason.NONE
             metadata = {
                 'unresolved_files': '',
                 'target_ref': pull_request.target_ref_parts,
                 'source_ref': pull_request.source_ref_parts,
             }
             if pull_request.last_merge_metadata:
                 metadata.update(pull_request.last_merge_metadata_parsed)
 
             if not possible and target_ref.type == 'branch':
                 # NOTE(marcink): case for mercurial multiple heads on branch
                 heads = target_vcs._heads(target_ref.name)
                 if len(heads) != 1:
                     heads = '\n,'.join(target_vcs._heads(target_ref.name))
                     metadata.update({
                         'heads': heads
                     })
 
             merge_state = MergeResponse(
                 possible, False, None, pull_request.last_merge_status, metadata=metadata)
 
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
         if reference.type in self.UPDATABLE_REF_TYPES:
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id
 
         refreshed_commit = vcs_repository.get_commit(name_or_id)
         refreshed_reference = Reference(
             reference.type, reference.name, refreshed_commit.raw_id)
         return refreshed_reference
 
     def _needs_merge_state_refresh(self, pull_request, target_reference):
         return not(
             pull_request.revisions and
             pull_request.revisions[0] == pull_request._last_merge_source_rev and
             target_reference.commit_id == pull_request._last_merge_target_rev)
 
     def _refresh_merge_state(self, pull_request, target_vcs, target_reference):
         workspace_id = self._workspace_id(pull_request)
         source_vcs = pull_request.source_repo.scm_instance()
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         merge_state = target_vcs.merge(
             repo_id, workspace_id,
             target_reference, source_vcs, pull_request.source_ref_parts,
             dry_run=True, use_rebase=use_rebase,
             close_branch=close_branch)
 
         # Do not store the response if there was an unknown error.
         if merge_state.failure_reason != MergeFailureReason.UNKNOWN:
             pull_request._last_merge_source_rev = \
                 pull_request.source_ref_parts.commit_id
             pull_request._last_merge_target_rev = target_reference.commit_id
             pull_request.last_merge_status = merge_state.failure_reason
             pull_request.last_merge_metadata = merge_state.metadata
 
             pull_request.shadow_merge_ref = merge_state.merge_ref
             Session().add(pull_request)
             Session().commit()
 
         return merge_state
 
     def _workspace_id(self, pull_request):
         workspace_id = 'pr-%s' % pull_request.pull_request_id
         return workspace_id
 
     def generate_repo_data(self, repo, commit_id=None, branch=None,
                            bookmark=None, translator=None):
         from rhodecode.model.repo import RepoModel
 
         all_refs, selected_ref = \
             self._get_repo_pullrequest_sources(
                 repo.scm_instance(), commit_id=commit_id,
                 branch=branch, bookmark=bookmark, translator=translator)
 
         refs_select2 = []
         for element in all_refs:
             children = [{'id': x[0], 'text': x[1]} for x in element[0]]
             refs_select2.append({'text': element[1], 'children': children})
 
         return {
             'user': {
                 'user_id': repo.user.user_id,
                 'username': repo.user.username,
                 'firstname': repo.user.first_name,
                 'lastname': repo.user.last_name,
                 'gravatar_link': h.gravatar_url(repo.user.email, 14),
             },
             'name': repo.repo_name,
             'link': RepoModel().get_url(repo),
             'description': h.chop_at_smart(repo.description_safe, '\n'),
             'refs': {
                 'all_refs': all_refs,
                 'selected_ref': selected_ref,
                 'select2_refs': refs_select2
             }
         }
 
     def generate_pullrequest_title(self, source, source_ref, target):
         return u'{source}#{at_ref} to {target}'.format(
             source=source,
             at_ref=source_ref,
             target=target,
         )
 
     def _cleanup_merge_workspace(self, pull_request):
         # Merging related cleanup
         repo_id = pull_request.target_repo.repo_id
         target_scm = pull_request.target_repo.scm_instance()
         workspace_id = self._workspace_id(pull_request)
 
         try:
             target_scm.cleanup_merge_workspace(repo_id, workspace_id)
         except NotImplementedError:
             pass
 
     def _get_repo_pullrequest_sources(
             self, repo, commit_id=None, branch=None, bookmark=None,
             translator=None):
         """
         Return a structure with repo's interesting commits, suitable for
         the selectors in pullrequest controller
 
         :param commit_id: a commit that must be in the list somehow
             and selected by default
         :param branch: a branch that must be in the list and selected
             by default - even if closed
         :param bookmark: a bookmark that must be in the list and selected
         """
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
         branch = safe_str(branch) if branch else None
         bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
         # order matters: first source that has commit_id in it will be selected
         sources = []
         sources.append(('book', repo.bookmarks.items(), _('Bookmarks'), bookmark))
         sources.append(('branch', repo.branches.items(), _('Branches'), branch))
 
         if commit_id:
             ref_commit = (h.short_id(commit_id), commit_id)
             sources.append(('rev', [ref_commit], _('Commit IDs'), commit_id))
 
         sources.append(
             ('branch', repo.branches_closed.items(), _('Closed Branches'), branch),
         )
 
         groups = []
 
         for group_key, ref_list, group_name, match in sources:
             group_refs = []
             for ref_name, ref_id in ref_list:
                 ref_key = u'{}:{}:{}'.format(group_key, ref_name, ref_id)
                 group_refs.append((ref_key, ref_name))
 
                 if not selected:
                     if set([commit_id, match]) & set([ref_id, ref_name]):
                         selected = ref_key
 
             if group_refs:
                 groups.append((group_refs, group_name))
 
         if not selected:
             ref = commit_id or branch or bookmark
             if ref:
                 raise CommitDoesNotExistError(
                     u'No commit refs could be found matching: {}'.format(ref))
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = u'branch:{}:{}'.format(
                     safe_str(repo.DEFAULT_BRANCH_NAME),
                     safe_str(repo.branches[repo.DEFAULT_BRANCH_NAME])
                 )
             elif repo.commit_ids:
                 # make the user select in this case
                 selected = None
             else:
                 raise EmptyRepositoryError()
         return groups, selected
 
     def get_diff(self, source_repo, source_ref_id, target_ref_id,
                  hide_whitespace_changes, diff_context):
 
         return self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
     def _get_diff_from_pr_or_version(
             self, source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context):
 
         target_commit = source_repo.get_commit(
             commit_id=safe_str(target_ref_id))
         source_commit = source_repo.get_commit(
             commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         if isinstance(source_repo, Repository):
             vcs_repo = source_repo.scm_instance()
         else:
             vcs_repo = source_repo
 
         # TODO: johbo: In the context of an update, we cannot reach
         # the old commit anymore with our normal mechanisms. It needs
         # some sort of special support in the vcs layer to avoid this
         # workaround.
         if (source_commit.raw_id == vcs_repo.EMPTY_COMMIT_ID and
                 vcs_repo.alias == 'git'):
             source_commit.raw_id = safe_str(source_ref_id)
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
                   safe_str(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit,
             ignore_whitespace=hide_whitespace_changes, context=diff_context)
         return vcs_diff
 
     def _is_merge_enabled(self, pull_request):
         return self._get_general_setting(
             pull_request, 'rhodecode_pr_merge_enabled')
 
     def _use_rebase_for_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_use_rebase_for_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_use_rebase_for_merging')
 
         return False
 
     def _user_name_for_merging(self, pull_request, user):
         env_user_name_attr = os.environ.get('RC_MERGE_USER_NAME_ATTR', '')
         if env_user_name_attr and hasattr(user, env_user_name_attr):
             user_name_attr = env_user_name_attr
         else:
             user_name_attr = 'short_contact'
 
         user_name = getattr(user, user_name_attr)
         return user_name
 
     def _close_branch_before_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_close_branch_before_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_close_branch_before_merging')
 
         return False
 
     def _get_general_setting(self, pull_request, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=pull_request.target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _log_audit_action(self, action, action_data, user, pull_request):
         audit_logger.store(
             action=action,
             action_data=action_data,
             user=user,
             repo=pull_request.target_repo)
 
     def get_reviewer_functions(self):
         """
         Fetches functions for validation and fetching default reviewers.
         If available we use the EE package, else we fallback to CE
         package functions
         """
         try:
             from rc_reviewers.utils import get_default_reviewers_data
             from rc_reviewers.utils import validate_default_reviewers
             from rc_reviewers.utils import validate_observers
         except ImportError:
             from rhodecode.apps.repository.utils import get_default_reviewers_data
             from rhodecode.apps.repository.utils import validate_default_reviewers
             from rhodecode.apps.repository.utils import validate_observers
 
         return get_default_reviewers_data, validate_default_reviewers, validate_observers
 
 
 class MergeCheck(object):
     """
     Perform Merge Checks and returns a check object which stores information
     about merge errors, and merge conditions
     """
     TODO_CHECK = 'todo'
     PERM_CHECK = 'perm'
     REVIEW_CHECK = 'review'
     MERGE_CHECK = 'merge'
     WIP_CHECK = 'wip'
 
     def __init__(self):
         self.review_status = None
         self.merge_possible = None
         self.merge_msg = ''
         self.merge_response = None
         self.failed = None
         self.errors = []
         self.error_details = OrderedDict()
         self.source_commit = AttributeDict()
         self.target_commit = AttributeDict()
         self.reviewers_count = 0
         self.observers_count = 0
 
     def __repr__(self):
         return '<MergeCheck(possible:{}, failed:{}, errors:{})>'.format(
             self.merge_possible, self.failed, self.errors)
 
     def push_error(self, error_type, message, error_key, details):
         self.failed = True
         self.errors.append([error_type, message])
         self.error_details[error_key] = dict(
             details=details,
             error_type=error_type,
             message=message
         )
 
     @classmethod
     def validate(cls, pull_request, auth_user, translator, fail_early=False,
                  force_shadow_repo_refresh=False):
         _ = translator
         merge_check = cls()
 
         # title has WIP:
         if pull_request.work_in_progress:
             log.debug("MergeCheck: cannot merge, title has wip: marker.")
 
             msg = _('WIP marker in title prevents from accidental merge.')
             merge_check.push_error('error', msg, cls.WIP_CHECK, pull_request.title)
             if fail_early:
                 return merge_check
 
         # permissions to merge
         user_allowed_to_merge = PullRequestModel().check_user_merge(pull_request, auth_user)
         if not user_allowed_to_merge:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('User `{}` not allowed to perform merge.').format(auth_user.username)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # permission to merge into the target branch
         target_commit_id = pull_request.target_ref_parts.commit_id
         if pull_request.target_ref_parts.type == 'branch':
             branch_name = pull_request.target_ref_parts.name
         else:
             # for mercurial we can always figure out the branch from the commit
             # in case of bookmark
             target_commit = pull_request.target_repo.get_commit(target_commit_id)
             branch_name = target_commit.branch
 
         rule, branch_perm = auth_user.get_rule_and_branch_permission(
             pull_request.target_repo.repo_name, branch_name)
         if branch_perm and branch_perm == 'branch.none':
             msg = _('Target branch `{}` changes rejected by rule {}.').format(
                 branch_name, rule)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # review status, must be always present
         review_status = pull_request.calculated_review_status()
         merge_check.review_status = review_status
         merge_check.reviewers_count = pull_request.reviewers_count
         merge_check.observers_count = pull_request.observers_count
 
         status_approved = review_status == ChangesetStatus.STATUS_APPROVED
         if not status_approved and merge_check.reviewers_count:
             log.debug("MergeCheck: cannot merge, approval is pending.")
             msg = _('Pull request reviewer approval is pending.')
 
             merge_check.push_error('warning', msg, cls.REVIEW_CHECK, review_status)
 
             if fail_early:
                 return merge_check
 
         # left over TODOs
         todos = CommentsModel().get_pull_request_unresolved_todos(pull_request)
         if todos:
             log.debug("MergeCheck: cannot merge, {} "
                       "unresolved TODOs left.".format(len(todos)))
 
             if len(todos) == 1:
                 msg = _('Cannot merge, {} TODO still not resolved.').format(
                     len(todos))
             else:
                 msg = _('Cannot merge, {} TODOs still not resolved.').format(
                     len(todos))
 
             merge_check.push_error('warning', msg, cls.TODO_CHECK, todos)
 
             if fail_early:
                 return merge_check
 
         # merge possible, here is the filesystem simulation + shadow repo
         merge_response, merge_status, msg = PullRequestModel().merge_status(
             pull_request, translator=translator,
             force_shadow_repo_refresh=force_shadow_repo_refresh)
 
         merge_check.merge_possible = merge_status
         merge_check.merge_msg = msg
         merge_check.merge_response = merge_response
 
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         try:
             source_commit, target_commit = PullRequestModel().get_flow_commits(pull_request)
             merge_check.source_commit.changed = source_ref_id != source_commit.raw_id
             merge_check.source_commit.ref_spec = pull_request.source_ref_parts
             merge_check.source_commit.current_raw_id = source_commit.raw_id
             merge_check.source_commit.previous_raw_id = source_ref_id
 
             merge_check.target_commit.changed = target_ref_id != target_commit.raw_id
             merge_check.target_commit.ref_spec = pull_request.target_ref_parts
             merge_check.target_commit.current_raw_id = target_commit.raw_id
             merge_check.target_commit.previous_raw_id = target_ref_id
         except (SourceRefMissing, TargetRefMissing):
             pass
 
         if not merge_status:
             log.debug("MergeCheck: cannot merge, pull request merge not possible.")
             merge_check.push_error('warning', msg, cls.MERGE_CHECK, None)
 
             if fail_early:
                 return merge_check
 
         log.debug('MergeCheck: is failed: %s', merge_check.failed)
         return merge_check
 
     @classmethod
     def get_merge_conditions(cls, pull_request, translator):
         _ = translator
         merge_details = {}
 
         model = PullRequestModel()
         use_rebase = model._use_rebase_for_merging(pull_request)
 
         if use_rebase:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: rebase')
             )
         else:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: explicit merge commit')
             )
 
         close_branch = model._close_branch_before_merging(pull_request)
         if close_branch:
             repo_type = pull_request.target_repo.repo_type
             close_msg = ''
             if repo_type == 'hg':
                 close_msg = _('Source branch will be closed before the merge.')
             elif repo_type == 'git':
                 close_msg = _('Source branch will be deleted after the merge.')
 
             merge_details['close_branch'] = dict(
                 details={},
                 message=close_msg
             )
 
         return merge_details
 
 
 @dataclasses.dataclass
 class ChangeTuple:
     added: list
     common: list
     removed: list
     total: list
 
 
 @dataclasses.dataclass
 class FileChangeTuple:
     added: list
     modified: list
     removed: list

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -1,223 +1,223 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
-from rhodecode.lib.pyramid_utils import get_app_config
+from rhodecode.lib.config_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests.server_utils import RcVCSServer
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests which need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     return vcsserver_factory(
         request, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
                 log_file=None, workers='2'):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         overrides.append({'server:main': {'port': vcsserver_port}})
 
         option_name = 'vcsserver_config_http'
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         server = RcVCSServer(config_file, log_file, workers)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def _use_log_level(config):
     level = config.getoption('test_loglevel') or 'critical'
     return level.upper()
 
 
 @pytest.fixture(scope='session')
 def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pyramid_config'
     log_level = _use_log_level(request.config)
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
             'cache_dir': '%(here)s/rc_data',
             'vcs.server': f'localhost:{vcsserver_port}',
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
             # off in the INI file.
             'vcs.start_server': 'false',
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '*',
             'app.service_api.token': 'service_secret_token',
         }},
 
         {'handler_console': {
             'class': 'StreamHandler',
             'args': '(sys.stderr,)',
             'level': log_level,
         }},
 
     ]
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def ini_settings(ini_config):
     ini_path = ini_config
     return get_app_config(ini_path)
 
 
 def get_available_port(min_port=40000, max_port=55555):
     from rhodecode.lib.utils2 import get_available_port as _get_port
     return _get_port(min_port, max_port)
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print(f'Using rhodecode port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print(f'Using vcsserver port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory():
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture()
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, ini_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, ini_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()

rhodecode/tests/fixture_mods/fixture_utils.py

@@ -1,1750 +1,1750 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import collections
 import datetime
 import os
 import re
 import pprint
 import shutil
 import socket
 import subprocess
 import time
 import uuid
 import dateutil.tz
 import logging
 import functools
 
 import mock
 import pyramid.testing
 import pytest
 import colander
 import requests
 import pyramid.paster
 
 import rhodecode
 import rhodecode.lib
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     PullRequest, PullRequestReviewers, Repository, RhodeCodeSetting, ChangesetStatus,
     RepoGroup, UserGroup, RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.integration import IntegrationModel
 from rhodecode.integrations import integration_type_registry
 from rhodecode.integrations.types.base import IntegrationTypeBase
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests import (
     login_user_session, get_new_dir, utils, TESTS_TMP_PATH,
     TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR2_LOGIN,
     TEST_USER_REGULAR_PASS)
 from rhodecode.tests.utils import CustomTestApp, set_anonymous_access
 from rhodecode.tests.fixture import Fixture
 from rhodecode.config import utils as config_utils
 
 log = logging.getLogger(__name__)
 
 
 def cmp(a, b):
     # backport cmp from python2 so we can still use it in the custom code in this module
     return (a > b) - (a < b)
 
 
 @pytest.fixture(scope='session', autouse=True)
 def activate_example_rcextensions(request):
     """
     Patch in an example rcextensions module which verifies passed in kwargs.
     """
     from rhodecode.config import rcextensions
 
     old_extensions = rhodecode.EXTENSIONS
     rhodecode.EXTENSIONS = rcextensions
     rhodecode.EXTENSIONS.calls = collections.defaultdict(list)
 
     @request.addfinalizer
     def cleanup():
         rhodecode.EXTENSIONS = old_extensions
 
 
 @pytest.fixture()
 def capture_rcextensions():
     """
     Returns the recorded calls to entry points in rcextensions.
     """
     calls = rhodecode.EXTENSIONS.calls
     calls.clear()
     # Note: At this moment, it is still the empty dict, but that will
     # be filled during the test run and since it is a reference this
     # is enough to make it work.
     return calls
 
 
 @pytest.fixture(scope='session')
 def http_environ_session():
     """
     Allow to use "http_environ" in session scope.
     """
     return plain_http_environ()
 
 
 def plain_http_host_stub():
     """
     Value of HTTP_HOST in the test run.
     """
     return 'example.com:80'
 
 
 @pytest.fixture()
 def http_host_stub():
     """
     Value of HTTP_HOST in the test run.
     """
     return plain_http_host_stub()
 
 
 def plain_http_host_only_stub():
     """
     Value of HTTP_HOST in the test run.
     """
     return plain_http_host_stub().split(':')[0]
 
 
 @pytest.fixture()
 def http_host_only_stub():
     """
     Value of HTTP_HOST in the test run.
     """
     return plain_http_host_only_stub()
 
 
 def plain_http_environ():
     """
     HTTP extra environ keys.
 
     User by the test application and as well for setting up the pylons
     environment. In the case of the fixture "app" it should be possible
     to override this for a specific test case.
     """
     return {
         'SERVER_NAME': plain_http_host_only_stub(),
         'SERVER_PORT': plain_http_host_stub().split(':')[1],
         'HTTP_HOST': plain_http_host_stub(),
         'HTTP_USER_AGENT': 'rc-test-agent',
         'REQUEST_METHOD': 'GET'
     }
 
 
 @pytest.fixture()
 def http_environ():
     """
     HTTP extra environ keys.
 
     User by the test application and as well for setting up the pylons
     environment. In the case of the fixture "app" it should be possible
     to override this for a specific test case.
     """
     return plain_http_environ()
 
 
 @pytest.fixture(scope='session')
 def baseapp(ini_config, vcsserver, http_environ_session):
-    from rhodecode.lib.pyramid_utils import get_app_config
+    from rhodecode.lib.config_utils import get_app_config
     from rhodecode.config.middleware import make_pyramid_app
 
     log.info("Using the RhodeCode configuration:{}".format(ini_config))
     pyramid.paster.setup_logging(ini_config)
 
     settings = get_app_config(ini_config)
     app = make_pyramid_app({'__file__': ini_config}, **settings)
 
     return app
 
 
 @pytest.fixture(scope='function')
 def app(request, config_stub, baseapp, http_environ):
     app = CustomTestApp(
         baseapp,
         extra_environ=http_environ)
     if request.cls:
         request.cls.app = app
     return app
 
 
 @pytest.fixture(scope='session')
 def app_settings(baseapp, ini_config):
     """
     Settings dictionary used to create the app.
 
     Parses the ini file and passes the result through the sanitize and apply
     defaults mechanism in `rhodecode.config.middleware`.
     """
     return baseapp.config.get_settings()
 
 
 @pytest.fixture(scope='session')
 def db_connection(ini_settings):
     # Initialize the database connection.
     config_utils.initialize_database(ini_settings)
 
 
 LoginData = collections.namedtuple('LoginData', ('csrf_token', 'user'))
 
 
 def _autologin_user(app, *args):
     session = login_user_session(app, *args)
     csrf_token = rhodecode.lib.auth.get_csrf_token(session)
     return LoginData(csrf_token, session['rhodecode_user'])
 
 
 @pytest.fixture()
 def autologin_user(app):
     """
     Utility fixture which makes sure that the admin user is logged in
     """
     return _autologin_user(app)
 
 
 @pytest.fixture()
 def autologin_regular_user(app):
     """
     Utility fixture which makes sure that the regular user is logged in
     """
     return _autologin_user(
         app, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
 
 
 @pytest.fixture(scope='function')
 def csrf_token(request, autologin_user):
     return autologin_user.csrf_token
 
 
 @pytest.fixture(scope='function')
 def xhr_header(request):
     return {'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest'}
 
 
 @pytest.fixture()
 def real_crypto_backend(monkeypatch):
     """
     Switch the production crypto backend on for this test.
 
     During the test run the crypto backend is replaced with a faster
     implementation based on the MD5 algorithm.
     """
     monkeypatch.setattr(rhodecode, 'is_test', False)
 
 
 @pytest.fixture(scope='class')
 def index_location(request, baseapp):
     index_location = baseapp.config.get_settings()['search.location']
     if request.cls:
         request.cls.index_location = index_location
     return index_location
 
 
 @pytest.fixture(scope='session', autouse=True)
 def tests_tmp_path(request):
     """
     Create temporary directory to be used during the test session.
     """
     if not os.path.exists(TESTS_TMP_PATH):
         os.makedirs(TESTS_TMP_PATH)
 
     if not request.config.getoption('--keep-tmp-path'):
         @request.addfinalizer
         def remove_tmp_path():
             shutil.rmtree(TESTS_TMP_PATH)
 
     return TESTS_TMP_PATH
 
 
 @pytest.fixture()
 def test_repo_group(request):
     """
     Create a temporary repository group, and destroy it after
     usage automatically
     """
     fixture = Fixture()
     repogroupid = 'test_repo_group_%s' % str(time.time()).replace('.', '')
     repo_group = fixture.create_repo_group(repogroupid)
 
     def _cleanup():
         fixture.destroy_repo_group(repogroupid)
 
     request.addfinalizer(_cleanup)
     return repo_group
 
 
 @pytest.fixture()
 def test_user_group(request):
     """
     Create a temporary user group, and destroy it after
     usage automatically
     """
     fixture = Fixture()
     usergroupid = 'test_user_group_%s' % str(time.time()).replace('.', '')
     user_group = fixture.create_user_group(usergroupid)
 
     def _cleanup():
         fixture.destroy_user_group(user_group)
 
     request.addfinalizer(_cleanup)
     return user_group
 
 
 @pytest.fixture(scope='session')
 def test_repo(request):
     container = TestRepoContainer()
     request.addfinalizer(container._cleanup)
     return container
 
 
 class TestRepoContainer(object):
     """
     Container for test repositories which are used read only.
 
     Repositories will be created on demand and re-used during the lifetime
     of this object.
 
     Usage to get the svn test repository "minimal"::
 
        test_repo = TestContainer()
        repo = test_repo('minimal', 'svn')
 
     """
 
     dump_extractors = {
         'git': utils.extract_git_repo_from_dump,
         'hg': utils.extract_hg_repo_from_dump,
         'svn': utils.extract_svn_repo_from_dump,
     }
 
     def __init__(self):
         self._cleanup_repos = []
         self._fixture = Fixture()
         self._repos = {}
 
     def __call__(self, dump_name, backend_alias, config=None):
         key = (dump_name, backend_alias)
         if key not in self._repos:
             repo = self._create_repo(dump_name, backend_alias, config)
             self._repos[key] = repo.repo_id
         return Repository.get(self._repos[key])
 
     def _create_repo(self, dump_name, backend_alias, config):
         repo_name = f'{backend_alias}-{dump_name}'
         backend = get_backend(backend_alias)
         dump_extractor = self.dump_extractors[backend_alias]
         repo_path = dump_extractor(dump_name, repo_name)
 
         vcs_repo = backend(repo_path, config=config)
         repo2db_mapper({repo_name: vcs_repo})
 
         repo = RepoModel().get_by_repo_name(repo_name)
         self._cleanup_repos.append(repo_name)
         return repo
 
     def _cleanup(self):
         for repo_name in reversed(self._cleanup_repos):
             self._fixture.destroy_repo(repo_name)
 
 
 def backend_base(request, backend_alias, baseapp, test_repo):
     if backend_alias not in request.config.getoption('--backends'):
         pytest.skip("Backend %s not selected." % (backend_alias, ))
 
     utils.check_xfail_backends(request.node, backend_alias)
     utils.check_skip_backends(request.node, backend_alias)
 
     repo_name = 'vcs_test_%s' % (backend_alias, )
     backend = Backend(
         alias=backend_alias,
         repo_name=repo_name,
         test_name=request.node.name,
         test_repo_container=test_repo)
     request.addfinalizer(backend.cleanup)
     return backend
 
 
 @pytest.fixture()
 def backend(request, backend_alias, baseapp, test_repo):
     """
     Parametrized fixture which represents a single backend implementation.
 
     It respects the option `--backends` to focus the test run on specific
     backend implementations.
 
     It also supports `pytest.mark.xfail_backends` to mark tests as failing
     for specific backends. This is intended as a utility for incremental
     development of a new backend implementation.
     """
     return backend_base(request, backend_alias, baseapp, test_repo)
 
 
 @pytest.fixture()
 def backend_git(request, baseapp, test_repo):
     return backend_base(request, 'git', baseapp, test_repo)
 
 
 @pytest.fixture()
 def backend_hg(request, baseapp, test_repo):
     return backend_base(request, 'hg', baseapp, test_repo)
 
 
 @pytest.fixture()
 def backend_svn(request, baseapp, test_repo):
     return backend_base(request, 'svn', baseapp, test_repo)
 
 
 @pytest.fixture()
 def backend_random(backend_git):
     """
     Use this to express that your tests need "a backend.
 
     A few of our tests need a backend, so that we can run the code. This
     fixture is intended to be used for such cases. It will pick one of the
     backends and run the tests.
 
     The fixture `backend` would run the test multiple times for each
     available backend which is a pure waste of time if the test is
     independent of the backend type.
     """
     # TODO: johbo: Change this to pick a random backend
     return backend_git
 
 
 @pytest.fixture()
 def backend_stub(backend_git):
     """
     Use this to express that your tests need a backend stub
 
     TODO: mikhail: Implement a real stub logic instead of returning
     a git backend
     """
     return backend_git
 
 
 @pytest.fixture()
 def repo_stub(backend_stub):
     """
     Use this to express that your tests need a repository stub
     """
     return backend_stub.create_repo()
 
 
 class Backend(object):
     """
     Represents the test configuration for one supported backend
 
     Provides easy access to different test repositories based on
     `__getitem__`. Such repositories will only be created once per test
     session.
     """
 
     invalid_repo_name = re.compile(r'[^0-9a-zA-Z]+')
     _master_repo = None
     _master_repo_path = ''
     _commit_ids = {}
 
     def __init__(self, alias, repo_name, test_name, test_repo_container):
         self.alias = alias
         self.repo_name = repo_name
         self._cleanup_repos = []
         self._test_name = test_name
         self._test_repo_container = test_repo_container
         # TODO: johbo: Used as a delegate interim. Not yet sure if Backend or
         # Fixture will survive in the end.
         self._fixture = Fixture()
 
     def __getitem__(self, key):
         return self._test_repo_container(key, self.alias)
 
     def create_test_repo(self, key, config=None):
         return self._test_repo_container(key, self.alias, config)
 
     @property
     def repo_id(self):
         # just fake some repo_id
         return self.repo.repo_id
 
     @property
     def repo(self):
         """
         Returns the "current" repository. This is the vcs_test repo or the
         last repo which has been created with `create_repo`.
         """
         from rhodecode.model.db import Repository
         return Repository.get_by_repo_name(self.repo_name)
 
     @property
     def default_branch_name(self):
         VcsRepository = get_backend(self.alias)
         return VcsRepository.DEFAULT_BRANCH_NAME
 
     @property
     def default_head_id(self):
         """
         Returns the default head id of the underlying backend.
 
         This will be the default branch name in case the backend does have a
         default branch. In the other cases it will point to a valid head
         which can serve as the base to create a new commit on top of it.
         """
         vcsrepo = self.repo.scm_instance()
         head_id = (
             vcsrepo.DEFAULT_BRANCH_NAME or
             vcsrepo.commit_ids[-1])
         return head_id
 
     @property
     def commit_ids(self):
         """
         Returns the list of commits for the last created repository
         """
         return self._commit_ids
 
     def create_master_repo(self, commits):
         """
         Create a repository and remember it as a template.
 
         This allows to easily create derived repositories to construct
         more complex scenarios for diff, compare and pull requests.
 
         Returns a commit map which maps from commit message to raw_id.
         """
         self._master_repo = self.create_repo(commits=commits)
         self._master_repo_path = self._master_repo.repo_full_path
 
         return self._commit_ids
 
     def create_repo(
             self, commits=None, number_of_commits=0, heads=None,
             name_suffix='', bare=False, **kwargs):
         """
         Create a repository and record it for later cleanup.
 
         :param commits: Optional. A sequence of dict instances.
            Will add a commit per entry to the new repository.
         :param number_of_commits: Optional. If set to a number, this number of
            commits will be added to the new repository.
         :param heads: Optional. Can be set to a sequence of of commit
            names which shall be pulled in from the master repository.
         :param name_suffix: adds special suffix to generated repo name
         :param bare: set a repo as bare (no checkout)
         """
         self.repo_name = self._next_repo_name() + name_suffix
         repo = self._fixture.create_repo(
             self.repo_name, repo_type=self.alias, bare=bare, **kwargs)
         self._cleanup_repos.append(repo.repo_name)
 
         commits = commits or [
             {'message': f'Commit {x} of {self.repo_name}'}
             for x in range(number_of_commits)]
         vcs_repo = repo.scm_instance()
         vcs_repo.count()
         self._add_commits_to_repo(vcs_repo, commits)
         if heads:
             self.pull_heads(repo, heads)
 
         return repo
 
     def pull_heads(self, repo, heads, do_fetch=False):
         """
         Make sure that repo contains all commits mentioned in `heads`
         """
         vcsrepo = repo.scm_instance()
         vcsrepo.config.clear_section('hooks')
         commit_ids = [self._commit_ids[h] for h in heads]
         if do_fetch:
             vcsrepo.fetch(self._master_repo_path, commit_ids=commit_ids)
         vcsrepo.pull(self._master_repo_path, commit_ids=commit_ids)
 
     def create_fork(self):
         repo_to_fork = self.repo_name
         self.repo_name = self._next_repo_name()
         repo = self._fixture.create_fork(repo_to_fork, self.repo_name)
         self._cleanup_repos.append(self.repo_name)
         return repo
 
     def new_repo_name(self, suffix=''):
         self.repo_name = self._next_repo_name() + suffix
         self._cleanup_repos.append(self.repo_name)
         return self.repo_name
 
     def _next_repo_name(self):
         return "%s_%s" % (
             self.invalid_repo_name.sub('_', self._test_name), len(self._cleanup_repos))
 
     def ensure_file(self, filename, content=b'Test content\n'):
         assert self._cleanup_repos, "Avoid writing into vcs_test repos"
         commits = [
             {'added': [
                 FileNode(filename, content=content),
             ]},
         ]
         self._add_commits_to_repo(self.repo.scm_instance(), commits)
 
     def enable_downloads(self):
         repo = self.repo
         repo.enable_downloads = True
         Session().add(repo)
         Session().commit()
 
     def cleanup(self):
         for repo_name in reversed(self._cleanup_repos):
             self._fixture.destroy_repo(repo_name)
 
     def _add_commits_to_repo(self, repo, commits):
         commit_ids = _add_commits_to_repo(repo, commits)
         if not commit_ids:
             return
         self._commit_ids = commit_ids
 
         # Creating refs for Git to allow fetching them from remote repository
         if self.alias == 'git':
             refs = {}
             for message in self._commit_ids:
                 cleanup_message = message.replace(' ', '')
                 ref_name = f'refs/test-refs/{cleanup_message}'
                 refs[ref_name] = self._commit_ids[message]
             self._create_refs(repo, refs)
 
     def _create_refs(self, repo, refs):
         for ref_name, ref_val in refs.items():
             repo.set_refs(ref_name, ref_val)
 
 
 class VcsBackend(object):
     """
     Represents the test configuration for one supported vcs backend.
     """
 
     invalid_repo_name = re.compile(r'[^0-9a-zA-Z]+')
 
     def __init__(self, alias, repo_path, test_name, test_repo_container):
         self.alias = alias
         self._repo_path = repo_path
         self._cleanup_repos = []
         self._test_name = test_name
         self._test_repo_container = test_repo_container
 
     def __getitem__(self, key):
         return self._test_repo_container(key, self.alias).scm_instance()
 
     def __repr__(self):
         return f'{self.__class__.__name__}(alias={self.alias}, repo={self._repo_path})'
 
     @property
     def repo(self):
         """
         Returns the "current" repository. This is the vcs_test repo of the last
         repo which has been created.
         """
         Repository = get_backend(self.alias)
         return Repository(self._repo_path)
 
     @property
     def backend(self):
         """
         Returns the backend implementation class.
         """
         return get_backend(self.alias)
 
     def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None,
                     bare=False):
         repo_name = self._next_repo_name()
         self._repo_path = get_new_dir(repo_name)
         repo_class = get_backend(self.alias)
         src_url = None
         if _clone_repo:
             src_url = _clone_repo.path
         repo = repo_class(self._repo_path, create=True, src_url=src_url, bare=bare)
         self._cleanup_repos.append(repo)
 
         commits = commits or [
             {'message': 'Commit %s of %s' % (x, repo_name)}
             for x in range(number_of_commits)]
         _add_commits_to_repo(repo, commits)
         return repo
 
     def clone_repo(self, repo):
         return self.create_repo(_clone_repo=repo)
 
     def cleanup(self):
         for repo in self._cleanup_repos:
             shutil.rmtree(repo.path)
 
     def new_repo_path(self):
         repo_name = self._next_repo_name()
         self._repo_path = get_new_dir(repo_name)
         return self._repo_path
 
     def _next_repo_name(self):
 
         return "{}_{}".format(
             self.invalid_repo_name.sub('_', self._test_name),
             len(self._cleanup_repos)
         )
 
     def add_file(self, repo, filename, content='Test content\n'):
         imc = repo.in_memory_commit
         imc.add(FileNode(safe_bytes(filename), content=safe_bytes(content)))
         imc.commit(
             message='Automatic commit from vcsbackend fixture',
             author='Automatic <[e-mail]>')
 
     def ensure_file(self, filename, content='Test content\n'):
         assert self._cleanup_repos, "Avoid writing into vcs_test repos"
         self.add_file(self.repo, filename, content)
 
 
 def vcsbackend_base(request, backend_alias, tests_tmp_path, baseapp, test_repo) -> VcsBackend:
     if backend_alias not in request.config.getoption('--backends'):
         pytest.skip("Backend %s not selected." % (backend_alias, ))
 
     utils.check_xfail_backends(request.node, backend_alias)
     utils.check_skip_backends(request.node, backend_alias)
 
     repo_name = f'vcs_test_{backend_alias}'
     repo_path = os.path.join(tests_tmp_path, repo_name)
     backend = VcsBackend(
         alias=backend_alias,
         repo_path=repo_path,
         test_name=request.node.name,
         test_repo_container=test_repo)
     request.addfinalizer(backend.cleanup)
     return backend
 
 
 @pytest.fixture()
 def vcsbackend(request, backend_alias, tests_tmp_path, baseapp, test_repo):
     """
     Parametrized fixture which represents a single vcs backend implementation.
 
     See the fixture `backend` for more details. This one implements the same
     concept, but on vcs level. So it does not provide model instances etc.
 
     Parameters are generated dynamically, see :func:`pytest_generate_tests`
     for how this works.
     """
     return vcsbackend_base(request, backend_alias, tests_tmp_path, baseapp, test_repo)
 
 
 @pytest.fixture()
 def vcsbackend_git(request, tests_tmp_path, baseapp, test_repo):
     return vcsbackend_base(request, 'git', tests_tmp_path, baseapp, test_repo)
 
 
 @pytest.fixture()
 def vcsbackend_hg(request, tests_tmp_path, baseapp, test_repo):
     return vcsbackend_base(request, 'hg', tests_tmp_path, baseapp, test_repo)
 
 
 @pytest.fixture()
 def vcsbackend_svn(request, tests_tmp_path, baseapp, test_repo):
     return vcsbackend_base(request, 'svn', tests_tmp_path, baseapp, test_repo)
 
 
 @pytest.fixture()
 def vcsbackend_stub(vcsbackend_git):
     """
     Use this to express that your test just needs a stub of a vcsbackend.
 
     Plan is to eventually implement an in-memory stub to speed tests up.
     """
     return vcsbackend_git
 
 
 def _add_commits_to_repo(vcs_repo, commits):
     commit_ids = {}
     if not commits:
         return commit_ids
 
     imc = vcs_repo.in_memory_commit
 
     for idx, commit in enumerate(commits):
         message = str(commit.get('message', f'Commit {idx}'))
 
         for node in commit.get('added', []):
             imc.add(FileNode(safe_bytes(node.path), content=node.content))
         for node in commit.get('changed', []):
             imc.change(FileNode(safe_bytes(node.path), content=node.content))
         for node in commit.get('removed', []):
             imc.remove(FileNode(safe_bytes(node.path)))
 
         parents = [
             vcs_repo.get_commit(commit_id=commit_ids[p])
             for p in commit.get('parents', [])]
 
         operations = ('added', 'changed', 'removed')
         if not any((commit.get(o) for o in operations)):
             imc.add(FileNode(b'file_%b' % safe_bytes(str(idx)), content=safe_bytes(message)))
 
         commit = imc.commit(
             message=message,
             author=str(commit.get('author', 'Automatic <[e-mail]>')),
             date=commit.get('date'),
             branch=commit.get('branch'),
             parents=parents)
 
         commit_ids[commit.message] = commit.raw_id
 
     return commit_ids
 
 
 @pytest.fixture()
 def reposerver(request):
     """
     Allows to serve a backend repository
     """
 
     repo_server = RepoServer()
     request.addfinalizer(repo_server.cleanup)
     return repo_server
 
 
 class RepoServer(object):
     """
     Utility to serve a local repository for the duration of a test case.
 
     Supports only Subversion so far.
     """
 
     url = None
 
     def __init__(self):
         self._cleanup_servers = []
 
     def serve(self, vcsrepo):
         if vcsrepo.alias != 'svn':
             raise TypeError("Backend %s not supported" % vcsrepo.alias)
 
         proc = subprocess.Popen(
             ['svnserve', '-d', '--foreground', '--listen-host', 'localhost',
              '--root', vcsrepo.path])
         self._cleanup_servers.append(proc)
         self.url = 'svn://localhost'
 
     def cleanup(self):
         for proc in self._cleanup_servers:
             proc.terminate()
 
 
 @pytest.fixture()
 def pr_util(backend, request, config_stub):
     """
     Utility for tests of models and for functional tests around pull requests.
 
     It gives an instance of :class:`PRTestUtility` which provides various
     utility methods around one pull request.
 
     This fixture uses `backend` and inherits its parameterization.
     """
 
     util = PRTestUtility(backend)
     request.addfinalizer(util.cleanup)
 
     return util
 
 
 class PRTestUtility(object):
 
     pull_request = None
     pull_request_id = None
     mergeable_patcher = None
     mergeable_mock = None
     notification_patcher = None
     commit_ids: dict
 
     def __init__(self, backend):
         self.backend = backend
 
     def create_pull_request(
             self, commits=None, target_head=None, source_head=None,
             revisions=None, approved=False, author=None, mergeable=False,
             enable_notifications=True, name_suffix='', reviewers=None, observers=None,
             title="Test", description="Description"):
         self.set_mergeable(mergeable)
         if not enable_notifications:
             # mock notification side effect
             self.notification_patcher = mock.patch(
                 'rhodecode.model.notification.NotificationModel.create')
             self.notification_patcher.start()
 
         if not self.pull_request:
             if not commits:
                 commits = [
                     {'message': 'c1'},
                     {'message': 'c2'},
                     {'message': 'c3'},
                 ]
                 target_head = 'c1'
                 source_head = 'c2'
                 revisions = ['c2']
 
             self.commit_ids = self.backend.create_master_repo(commits)
             self.target_repository = self.backend.create_repo(
                 heads=[target_head], name_suffix=name_suffix)
             self.source_repository = self.backend.create_repo(
                 heads=[source_head], name_suffix=name_suffix)
             self.author = author or UserModel().get_by_username(
                 TEST_USER_ADMIN_LOGIN)
 
             model = PullRequestModel()
             self.create_parameters = {
                 'created_by': self.author,
                 'source_repo': self.source_repository.repo_name,
                 'source_ref': self._default_branch_reference(source_head),
                 'target_repo': self.target_repository.repo_name,
                 'target_ref': self._default_branch_reference(target_head),
                 'revisions': [self.commit_ids[r] for r in revisions],
                 'reviewers': reviewers or self._get_reviewers(),
                 'observers': observers or self._get_observers(),
                 'title': title,
                 'description': description,
             }
             self.pull_request = model.create(**self.create_parameters)
             assert model.get_versions(self.pull_request) == []
 
             self.pull_request_id = self.pull_request.pull_request_id
 
             if approved:
                 self.approve()
 
             Session().add(self.pull_request)
             Session().commit()
 
         return self.pull_request
 
     def approve(self):
         self.create_status_votes(
             ChangesetStatus.STATUS_APPROVED,
             *self.pull_request.reviewers)
 
     def close(self):
         PullRequestModel().close_pull_request(self.pull_request, self.author)
 
     def _default_branch_reference(self, commit_message, branch: str = None) -> str:
         default_branch = branch or self.backend.default_branch_name
         message = self.commit_ids[commit_message]
         reference = f'branch:{default_branch}:{message}'
 
         return reference
 
     def _get_reviewers(self):
         role = PullRequestReviewers.ROLE_REVIEWER
         return [
             (TEST_USER_REGULAR_LOGIN, ['default1'], False, role, []),
             (TEST_USER_REGULAR2_LOGIN, ['default2'], False, role, []),
         ]
 
     def _get_observers(self):
         return [
 
         ]
 
     def update_source_repository(self, head=None, do_fetch=False):
         heads = [head or 'c3']
         self.backend.pull_heads(self.source_repository, heads=heads, do_fetch=do_fetch)
 
     def update_target_repository(self, head=None, do_fetch=False):
         heads = [head or 'c3']
         self.backend.pull_heads(self.target_repository, heads=heads, do_fetch=do_fetch)
 
     def set_pr_target_ref(self, ref_type: str = "branch", ref_name: str = "branch", ref_commit_id: str = "") -> str:
         full_ref = f"{ref_type}:{ref_name}:{ref_commit_id}"
         self.pull_request.target_ref = full_ref
         return full_ref
 
     def set_pr_source_ref(self, ref_type: str = "branch", ref_name: str = "branch", ref_commit_id: str = "") -> str:
         full_ref = f"{ref_type}:{ref_name}:{ref_commit_id}"
         self.pull_request.source_ref = full_ref
         return full_ref
 
     def add_one_commit(self, head=None):
         self.update_source_repository(head=head)
         old_commit_ids = set(self.pull_request.revisions)
         PullRequestModel().update_commits(self.pull_request, self.pull_request.author)
         commit_ids = set(self.pull_request.revisions)
         new_commit_ids = commit_ids - old_commit_ids
         assert len(new_commit_ids) == 1
         return new_commit_ids.pop()
 
     def remove_one_commit(self):
         assert len(self.pull_request.revisions) == 2
         source_vcs = self.source_repository.scm_instance()
         removed_commit_id = source_vcs.commit_ids[-1]
 
         # TODO: johbo: Git and Mercurial have an inconsistent vcs api here,
         # remove the if once that's sorted out.
         if self.backend.alias == "git":
             kwargs = {'branch_name': self.backend.default_branch_name}
         else:
             kwargs = {}
         source_vcs.strip(removed_commit_id, **kwargs)
 
         PullRequestModel().update_commits(self.pull_request, self.pull_request.author)
         assert len(self.pull_request.revisions) == 1
         return removed_commit_id
 
     def create_comment(self, linked_to=None):
         comment = CommentsModel().create(
             text="Test comment",
             repo=self.target_repository.repo_name,
             user=self.author,
             pull_request=self.pull_request)
         assert comment.pull_request_version_id is None
 
         if linked_to:
             PullRequestModel()._link_comments_to_version(linked_to)
 
         return comment
 
     def create_inline_comment(
             self, linked_to=None, line_no='n1', file_path='file_1'):
         comment = CommentsModel().create(
             text="Test comment",
             repo=self.target_repository.repo_name,
             user=self.author,
             line_no=line_no,
             f_path=file_path,
             pull_request=self.pull_request)
         assert comment.pull_request_version_id is None
 
         if linked_to:
             PullRequestModel()._link_comments_to_version(linked_to)
 
         return comment
 
     def create_version_of_pull_request(self):
         pull_request = self.create_pull_request()
         version = PullRequestModel()._create_version_from_snapshot(
             pull_request)
         return version
 
     def create_status_votes(self, status, *reviewers):
         for reviewer in reviewers:
             ChangesetStatusModel().set_status(
                 repo=self.pull_request.target_repo,
                 status=status,
                 user=reviewer.user_id,
                 pull_request=self.pull_request)
 
     def set_mergeable(self, value):
         if not self.mergeable_patcher:
             self.mergeable_patcher = mock.patch.object(
                 VcsSettingsModel, 'get_general_settings')
             self.mergeable_mock = self.mergeable_patcher.start()
         self.mergeable_mock.return_value = {
             'rhodecode_pr_merge_enabled': value}
 
     def cleanup(self):
         # In case the source repository is already cleaned up, the pull
         # request will already be deleted.
         pull_request = PullRequest().get(self.pull_request_id)
         if pull_request:
             PullRequestModel().delete(pull_request, pull_request.author)
             Session().commit()
 
         if self.notification_patcher:
             self.notification_patcher.stop()
 
         if self.mergeable_patcher:
             self.mergeable_patcher.stop()
 
 
 @pytest.fixture()
 def user_admin(baseapp):
     """
     Provides the default admin test user as an instance of `db.User`.
     """
     user = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
     return user
 
 
 @pytest.fixture()
 def user_regular(baseapp):
     """
     Provides the default regular test user as an instance of `db.User`.
     """
     user = UserModel().get_by_username(TEST_USER_REGULAR_LOGIN)
     return user
 
 
 @pytest.fixture()
 def user_util(request, db_connection):
     """
     Provides a wired instance of `UserUtility` with integrated cleanup.
     """
     utility = UserUtility(test_name=request.node.name)
     request.addfinalizer(utility.cleanup)
     return utility
 
 
 # TODO: johbo: Split this up into utilities per domain or something similar
 class UserUtility(object):
 
     def __init__(self, test_name="test"):
         self._test_name = self._sanitize_name(test_name)
         self.fixture = Fixture()
         self.repo_group_ids = []
         self.repos_ids = []
         self.user_ids = []
         self.user_group_ids = []
         self.user_repo_permission_ids = []
         self.user_group_repo_permission_ids = []
         self.user_repo_group_permission_ids = []
         self.user_group_repo_group_permission_ids = []
         self.user_user_group_permission_ids = []
         self.user_group_user_group_permission_ids = []
         self.user_permissions = []
 
     def _sanitize_name(self, name):
         for char in ['[', ']']:
             name = name.replace(char, '_')
         return name
 
     def create_repo_group(
             self, owner=TEST_USER_ADMIN_LOGIN, auto_cleanup=True):
         group_name = "{prefix}_repogroup_{count}".format(
             prefix=self._test_name,
             count=len(self.repo_group_ids))
         repo_group = self.fixture.create_repo_group(
             group_name, cur_user=owner)
         if auto_cleanup:
             self.repo_group_ids.append(repo_group.group_id)
         return repo_group
 
     def create_repo(self, owner=TEST_USER_ADMIN_LOGIN, parent=None,
                     auto_cleanup=True, repo_type='hg', bare=False):
         repo_name = "{prefix}_repository_{count}".format(
             prefix=self._test_name,
             count=len(self.repos_ids))
 
         repository = self.fixture.create_repo(
             repo_name, cur_user=owner, repo_group=parent, repo_type=repo_type, bare=bare)
         if auto_cleanup:
             self.repos_ids.append(repository.repo_id)
         return repository
 
     def create_user(self, auto_cleanup=True, **kwargs):
         user_name = "{prefix}_user_{count}".format(
             prefix=self._test_name,
             count=len(self.user_ids))
         user = self.fixture.create_user(user_name, **kwargs)
         if auto_cleanup:
             self.user_ids.append(user.user_id)
         return user
 
     def create_additional_user_email(self, user, email):
         uem = self.fixture.create_additional_user_email(user=user, email=email)
         return uem
 
     def create_user_with_group(self):
         user = self.create_user()
         user_group = self.create_user_group(members=[user])
         return user, user_group
 
     def create_user_group(self, owner=TEST_USER_ADMIN_LOGIN, members=None,
                           auto_cleanup=True, **kwargs):
         group_name = "{prefix}_usergroup_{count}".format(
             prefix=self._test_name,
             count=len(self.user_group_ids))
         user_group = self.fixture.create_user_group(
             group_name, cur_user=owner, **kwargs)
 
         if auto_cleanup:
             self.user_group_ids.append(user_group.users_group_id)
         if members:
             for user in members:
                 UserGroupModel().add_user_to_group(user_group, user)
         return user_group
 
     def grant_user_permission(self, user_name, permission_name):
         self.inherit_default_user_permissions(user_name, False)
         self.user_permissions.append((user_name, permission_name))
 
     def grant_user_permission_to_repo_group(
             self, repo_group, user, permission_name):
         permission = RepoGroupModel().grant_user_permission(
             repo_group, user, permission_name)
         self.user_repo_group_permission_ids.append(
             (repo_group.group_id, user.user_id))
         return permission
 
     def grant_user_group_permission_to_repo_group(
             self, repo_group, user_group, permission_name):
         permission = RepoGroupModel().grant_user_group_permission(
             repo_group, user_group, permission_name)
         self.user_group_repo_group_permission_ids.append(
             (repo_group.group_id, user_group.users_group_id))
         return permission
 
     def grant_user_permission_to_repo(
             self, repo, user, permission_name):
         permission = RepoModel().grant_user_permission(
             repo, user, permission_name)
         self.user_repo_permission_ids.append(
             (repo.repo_id, user.user_id))
         return permission
 
     def grant_user_group_permission_to_repo(
             self, repo, user_group, permission_name):
         permission = RepoModel().grant_user_group_permission(
             repo, user_group, permission_name)
         self.user_group_repo_permission_ids.append(
             (repo.repo_id, user_group.users_group_id))
         return permission
 
     def grant_user_permission_to_user_group(
             self, target_user_group, user, permission_name):
         permission = UserGroupModel().grant_user_permission(
             target_user_group, user, permission_name)
         self.user_user_group_permission_ids.append(
             (target_user_group.users_group_id, user.user_id))
         return permission
 
     def grant_user_group_permission_to_user_group(
             self, target_user_group, user_group, permission_name):
         permission = UserGroupModel().grant_user_group_permission(
             target_user_group, user_group, permission_name)
         self.user_group_user_group_permission_ids.append(
             (target_user_group.users_group_id, user_group.users_group_id))
         return permission
 
     def revoke_user_permission(self, user_name, permission_name):
         self.inherit_default_user_permissions(user_name, True)
         UserModel().revoke_perm(user_name, permission_name)
 
     def inherit_default_user_permissions(self, user_name, value):
         user = UserModel().get_by_username(user_name)
         user.inherit_default_permissions = value
         Session().add(user)
         Session().commit()
 
     def cleanup(self):
         self._cleanup_permissions()
         self._cleanup_repos()
         self._cleanup_repo_groups()
         self._cleanup_user_groups()
         self._cleanup_users()
 
     def _cleanup_permissions(self):
         if self.user_permissions:
             for user_name, permission_name in self.user_permissions:
                 self.revoke_user_permission(user_name, permission_name)
 
         for permission in self.user_repo_permission_ids:
             RepoModel().revoke_user_permission(*permission)
 
         for permission in self.user_group_repo_permission_ids:
             RepoModel().revoke_user_group_permission(*permission)
 
         for permission in self.user_repo_group_permission_ids:
             RepoGroupModel().revoke_user_permission(*permission)
 
         for permission in self.user_group_repo_group_permission_ids:
             RepoGroupModel().revoke_user_group_permission(*permission)
 
         for permission in self.user_user_group_permission_ids:
             UserGroupModel().revoke_user_permission(*permission)
 
         for permission in self.user_group_user_group_permission_ids:
             UserGroupModel().revoke_user_group_permission(*permission)
 
     def _cleanup_repo_groups(self):
         def _repo_group_compare(first_group_id, second_group_id):
             """
             Gives higher priority to the groups with the most complex paths
             """
             first_group = RepoGroup.get(first_group_id)
             second_group = RepoGroup.get(second_group_id)
             first_group_parts = (
                 len(first_group.group_name.split('/')) if first_group else 0)
             second_group_parts = (
                 len(second_group.group_name.split('/')) if second_group else 0)
             return cmp(second_group_parts, first_group_parts)
 
         sorted_repo_group_ids = sorted(
             self.repo_group_ids, key=functools.cmp_to_key(_repo_group_compare))
         for repo_group_id in sorted_repo_group_ids:
             self.fixture.destroy_repo_group(repo_group_id)
 
     def _cleanup_repos(self):
         sorted_repos_ids = sorted(self.repos_ids)
         for repo_id in sorted_repos_ids:
             self.fixture.destroy_repo(repo_id)
 
     def _cleanup_user_groups(self):
         def _user_group_compare(first_group_id, second_group_id):
             """
             Gives higher priority to the groups with the most complex paths
             """
             first_group = UserGroup.get(first_group_id)
             second_group = UserGroup.get(second_group_id)
             first_group_parts = (
                 len(first_group.users_group_name.split('/'))
                 if first_group else 0)
             second_group_parts = (
                 len(second_group.users_group_name.split('/'))
                 if second_group else 0)
             return cmp(second_group_parts, first_group_parts)
 
         sorted_user_group_ids = sorted(
             self.user_group_ids, key=functools.cmp_to_key(_user_group_compare))
         for user_group_id in sorted_user_group_ids:
             self.fixture.destroy_user_group(user_group_id)
 
     def _cleanup_users(self):
         for user_id in self.user_ids:
             self.fixture.destroy_user(user_id)
 
 
 @pytest.fixture(scope='session')
 def testrun():
     return {
         'uuid': uuid.uuid4(),
         'start': datetime.datetime.utcnow().isoformat(),
         'timestamp': int(time.time()),
     }
 
 
 class AppenlightClient(object):
 
     url_template = '{url}?protocol_version=0.5'
 
     def __init__(
             self, url, api_key, add_server=True, add_timestamp=True,
             namespace=None, request=None, testrun=None):
         self.url = self.url_template.format(url=url)
         self.api_key = api_key
         self.add_server = add_server
         self.add_timestamp = add_timestamp
         self.namespace = namespace
         self.request = request
         self.server = socket.getfqdn(socket.gethostname())
         self.tags_before = {}
         self.tags_after = {}
         self.stats = []
         self.testrun = testrun or {}
 
     def tag_before(self, tag, value):
         self.tags_before[tag] = value
 
     def tag_after(self, tag, value):
         self.tags_after[tag] = value
 
     def collect(self, data):
         if self.add_server:
             data.setdefault('server', self.server)
         if self.add_timestamp:
             data.setdefault('date', datetime.datetime.utcnow().isoformat())
         if self.namespace:
             data.setdefault('namespace', self.namespace)
         if self.request:
             data.setdefault('request', self.request)
         self.stats.append(data)
 
     def send_stats(self):
         tags = [
             ('testrun', self.request),
             ('testrun.start', self.testrun['start']),
             ('testrun.timestamp', self.testrun['timestamp']),
             ('test', self.namespace),
         ]
         for key, value in self.tags_before.items():
             tags.append((key + '.before', value))
             try:
                 delta = self.tags_after[key] - value
                 tags.append((key + '.delta', delta))
             except Exception:
                 pass
         for key, value in self.tags_after.items():
             tags.append((key + '.after', value))
         self.collect({
             'message': "Collected tags",
             'tags': tags,
         })
 
         response = requests.post(
             self.url,
             headers={
                 'X-appenlight-api-key': self.api_key},
             json=self.stats,
         )
 
         if not response.status_code == 200:
             pprint.pprint(self.stats)
             print(response.headers)
             print(response.text)
             raise Exception('Sending to appenlight failed')
 
 
 @pytest.fixture()
 def gist_util(request, db_connection):
     """
     Provides a wired instance of `GistUtility` with integrated cleanup.
     """
     utility = GistUtility()
     request.addfinalizer(utility.cleanup)
     return utility
 
 
 class GistUtility(object):
     def __init__(self):
         self.fixture = Fixture()
         self.gist_ids = []
 
     def create_gist(self, **kwargs):
         gist = self.fixture.create_gist(**kwargs)
         self.gist_ids.append(gist.gist_id)
         return gist
 
     def cleanup(self):
         for id_ in self.gist_ids:
             self.fixture.destroy_gists(str(id_))
 
 
 @pytest.fixture()
 def enabled_backends(request):
     backends = request.config.option.backends
     return backends[:]
 
 
 @pytest.fixture()
 def settings_util(request, db_connection):
     """
     Provides a wired instance of `SettingsUtility` with integrated cleanup.
     """
     utility = SettingsUtility()
     request.addfinalizer(utility.cleanup)
     return utility
 
 
 class SettingsUtility(object):
     def __init__(self):
         self.rhodecode_ui_ids = []
         self.rhodecode_setting_ids = []
         self.repo_rhodecode_ui_ids = []
         self.repo_rhodecode_setting_ids = []
 
     def create_repo_rhodecode_ui(
             self, repo, section, value, key=None, active=True, cleanup=True):
         key = key or sha1_safe(f'{section}{value}{repo.repo_id}')
 
         setting = RepoRhodeCodeUi()
         setting.repository_id = repo.repo_id
         setting.ui_section = section
         setting.ui_value = value
         setting.ui_key = key
         setting.ui_active = active
         Session().add(setting)
         Session().commit()
 
         if cleanup:
             self.repo_rhodecode_ui_ids.append(setting.ui_id)
         return setting
 
     def create_rhodecode_ui(
             self, section, value, key=None, active=True, cleanup=True):
         key = key or sha1_safe(f'{section}{value}')
 
         setting = RhodeCodeUi()
         setting.ui_section = section
         setting.ui_value = value
         setting.ui_key = key
         setting.ui_active = active
         Session().add(setting)
         Session().commit()
 
         if cleanup:
             self.rhodecode_ui_ids.append(setting.ui_id)
         return setting
 
     def create_repo_rhodecode_setting(
             self, repo, name, value, type_, cleanup=True):
         setting = RepoRhodeCodeSetting(
             repo.repo_id, key=name, val=value, type=type_)
         Session().add(setting)
         Session().commit()
 
         if cleanup:
             self.repo_rhodecode_setting_ids.append(setting.app_settings_id)
         return setting
 
     def create_rhodecode_setting(self, name, value, type_, cleanup=True):
         setting = RhodeCodeSetting(key=name, val=value, type=type_)
         Session().add(setting)
         Session().commit()
 
         if cleanup:
             self.rhodecode_setting_ids.append(setting.app_settings_id)
 
         return setting
 
     def cleanup(self):
         for id_ in self.rhodecode_ui_ids:
             setting = RhodeCodeUi.get(id_)
             Session().delete(setting)
 
         for id_ in self.rhodecode_setting_ids:
             setting = RhodeCodeSetting.get(id_)
             Session().delete(setting)
 
         for id_ in self.repo_rhodecode_ui_ids:
             setting = RepoRhodeCodeUi.get(id_)
             Session().delete(setting)
 
         for id_ in self.repo_rhodecode_setting_ids:
             setting = RepoRhodeCodeSetting.get(id_)
             Session().delete(setting)
 
         Session().commit()
 
 
 @pytest.fixture()
 def no_notifications(request):
     notification_patcher = mock.patch(
         'rhodecode.model.notification.NotificationModel.create')
     notification_patcher.start()
     request.addfinalizer(notification_patcher.stop)
 
 
 @pytest.fixture(scope='session')
 def repeat(request):
     """
     The number of repetitions is based on this fixture.
 
     Slower calls may divide it by 10 or 100. It is chosen in a way so that the
     tests are not too slow in our default test suite.
     """
     return request.config.getoption('--repeat')
 
 
 @pytest.fixture()
 def rhodecode_fixtures():
     return Fixture()
 
 
 @pytest.fixture()
 def context_stub():
     """
     Stub context object.
     """
     context = pyramid.testing.DummyResource()
     return context
 
 
 @pytest.fixture()
 def request_stub():
     """
     Stub request object.
     """
     from rhodecode.lib.base import bootstrap_request
     request = bootstrap_request(scheme='https')
     return request
 
 
 @pytest.fixture()
 def config_stub(request, request_stub):
     """
     Set up pyramid.testing and return the Configurator.
     """
     from rhodecode.lib.base import bootstrap_config
     config = bootstrap_config(request=request_stub)
 
     @request.addfinalizer
     def cleanup():
         pyramid.testing.tearDown()
 
     return config
 
 
 @pytest.fixture()
 def StubIntegrationType():
     class _StubIntegrationType(IntegrationTypeBase):
         """ Test integration type class """
 
         key = 'test'
         display_name = 'Test integration type'
         description = 'A test integration type for testing'
 
         @classmethod
         def icon(cls):
             return 'test_icon_html_image'
 
         def __init__(self, settings):
             super(_StubIntegrationType, self).__init__(settings)
             self.sent_events = []  # for testing
 
         def send_event(self, event):
             self.sent_events.append(event)
 
         def settings_schema(self):
             class SettingsSchema(colander.Schema):
                 test_string_field = colander.SchemaNode(
                     colander.String(),
                     missing=colander.required,
                     title='test string field',
                 )
                 test_int_field = colander.SchemaNode(
                     colander.Int(),
                     title='some integer setting',
                 )
             return SettingsSchema()
 
 
     integration_type_registry.register_integration_type(_StubIntegrationType)
     return _StubIntegrationType
 
 
 @pytest.fixture()
 def stub_integration_settings():
     return {
         'test_string_field': 'some data',
         'test_int_field': 100,
     }
 
 
 @pytest.fixture()
 def repo_integration_stub(request, repo_stub, StubIntegrationType,
         stub_integration_settings):
     integration = IntegrationModel().create(
         StubIntegrationType, settings=stub_integration_settings, enabled=True,
         name='test repo integration',
         repo=repo_stub, repo_group=None, child_repos_only=None)
 
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
 
     return integration
 
 
 @pytest.fixture()
 def repogroup_integration_stub(request, test_repo_group, StubIntegrationType,
     stub_integration_settings):
     integration = IntegrationModel().create(
         StubIntegrationType, settings=stub_integration_settings, enabled=True,
         name='test repogroup integration',
         repo=None, repo_group=test_repo_group, child_repos_only=True)
 
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
 
     return integration
 
 
 @pytest.fixture()
 def repogroup_recursive_integration_stub(request, test_repo_group,
     StubIntegrationType, stub_integration_settings):
     integration = IntegrationModel().create(
         StubIntegrationType, settings=stub_integration_settings, enabled=True,
         name='test recursive repogroup integration',
         repo=None, repo_group=test_repo_group, child_repos_only=False)
 
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
 
     return integration
 
 
 @pytest.fixture()
 def global_integration_stub(request, StubIntegrationType,
     stub_integration_settings):
     integration = IntegrationModel().create(
         StubIntegrationType, settings=stub_integration_settings, enabled=True,
         name='test global integration',
         repo=None, repo_group=None, child_repos_only=None)
 
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
 
     return integration
 
 
 @pytest.fixture()
 def root_repos_integration_stub(request, StubIntegrationType,
     stub_integration_settings):
     integration = IntegrationModel().create(
         StubIntegrationType, settings=stub_integration_settings, enabled=True,
         name='test global integration',
         repo=None, repo_group=None, child_repos_only=True)
 
     @request.addfinalizer
     def cleanup():
         IntegrationModel().delete(integration)
 
     return integration
 
 
 @pytest.fixture()
 def local_dt_to_utc():
     def _factory(dt):
         return dt.replace(tzinfo=dateutil.tz.tzlocal()).astimezone(
             dateutil.tz.tzutc()).replace(tzinfo=None)
     return _factory
 
 
 @pytest.fixture()
 def disable_anonymous_user(request, baseapp):
     set_anonymous_access(False)
 
     @request.addfinalizer
     def cleanup():
         set_anonymous_access(True)
 
 
 @pytest.fixture(scope='module')
 def rc_fixture(request):
     return Fixture()
 
 
 @pytest.fixture()
 def repo_groups(request):
     fixture = Fixture()
 
     session = Session()
     zombie_group = fixture.create_repo_group('zombie')
     parent_group = fixture.create_repo_group('parent')
     child_group = fixture.create_repo_group('parent/child')
     groups_in_db = session.query(RepoGroup).all()
     assert len(groups_in_db) == 3
     assert child_group.group_parent_id == parent_group.group_id
 
     @request.addfinalizer
     def cleanup():
         fixture.destroy_repo_group(zombie_group)
         fixture.destroy_repo_group(child_group)
         fixture.destroy_repo_group(parent_group)
 
     return zombie_group, parent_group, child_group

rhodecode/tests/lib/test_hooks_daemon.py

@@ -1,355 +1,359 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import io
 
 import mock
 import msgpack
 import pytest
 import tempfile
 
-from rhodecode.lib import hooks_daemon
+from rhodecode.lib.hook_daemon import http_hooks_deamon
+from rhodecode.lib.hook_daemon import celery_hooks_deamon
+from rhodecode.lib.hook_daemon import hook_module
+from rhodecode.lib.hook_daemon import base as hook_base
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.tests.utils import assert_message_in_log
 from rhodecode.lib.ext_json import json
 
-test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
+test_proto = http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
 
 
 class TestHooks(object):
     def test_hooks_can_be_used_as_a_context_processor(self):
-        hooks = hooks_daemon.Hooks()
+        hooks = hook_module.Hooks()
         with hooks as return_value:
             pass
         assert hooks == return_value
 
 
 class TestHooksHttpHandler(object):
     def test_read_request_parses_method_name_and_arguments(self):
         data = {
             'method': 'test',
             'extras': {
                 'param1': 1,
                 'param2': 'a'
             }
         }
         request = self._generate_post_request(data)
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, data['method'], create=True, return_value=1)
+            hook_module.Hooks, data['method'], create=True, return_value=1)
 
         with hooks_patcher as hooks_mock:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             MockServer(handler, request)
 
         hooks_mock.assert_called_once_with(data['extras'])
 
     def test_hooks_serialized_result_is_returned(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
         hook_result = {
             'first': 'one',
             'second': 2
         }
         extras = {}
 
         # patching our _read to return test method and proto used
         read_patcher = mock.patch.object(
-            hooks_daemon.HooksHttpHandler, '_read_request',
+            http_hooks_deamon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, extras))
 
         # patch Hooks instance to return hook_result data on 'test' call
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, rpc_method, create=True,
+            hook_module.Hooks, rpc_method, create=True,
             return_value=hook_result)
 
         with read_patcher, hooks_patcher:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
 
-        expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result)
+        expected_result = http_hooks_deamon.HooksHttpHandler.serialize_data(hook_result)
 
         server.request.output_stream.seek(0)
         assert server.request.output_stream.readlines()[-1] == expected_result
 
     def test_exception_is_returned_in_response(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
 
         read_patcher = mock.patch.object(
-            hooks_daemon.HooksHttpHandler, '_read_request',
+            http_hooks_deamon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, {}))
 
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, rpc_method, create=True,
+            hook_module.Hooks, rpc_method, create=True,
             side_effect=Exception('Test exception'))
 
         with read_patcher, hooks_patcher:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
 
         server.request.output_stream.seek(0)
         data = server.request.output_stream.readlines()
         msgpack_data = b''.join(data[5:])
-        org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data)
+        org_exc = http_hooks_deamon.HooksHttpHandler.deserialize_data(msgpack_data)
         expected_result = {
             'exception': 'Exception',
             'exception_traceback': org_exc['exception_traceback'],
             'exception_args': ['Test exception']
         }
         assert org_exc == expected_result
 
     def test_log_message_writes_to_debug_log(self, caplog):
         ip_port = ('0.0.0.0', 8888)
-        handler = hooks_daemon.HooksHttpHandler(
-            MockRequest('POST /'), ip_port, mock.Mock())
+        handler = http_hooks_deamon.HooksHttpHandler(MockRequest('POST /'), ip_port, mock.Mock())
         fake_date = '1/Nov/2015 00:00:00'
         date_patcher = mock.patch.object(
             handler, 'log_date_time_string', return_value=fake_date)
 
         with date_patcher, caplog.at_level(logging.DEBUG):
             handler.log_message('Some message %d, %s', 123, 'string')
 
         expected_message = f"HOOKS: client={ip_port} - - [{fake_date}] Some message 123, string"
 
         assert_message_in_log(
             caplog.records, expected_message,
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def _generate_post_request(self, data, proto=test_proto):
-        if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
+        if proto == http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
             payload = msgpack.packb(data)
         else:
             payload = json.dumps(data)
 
         return b'POST / HTTP/1.0\nContent-Length: %d\n\n%b' % (
             len(payload), payload)
 
 
 class ThreadedHookCallbackDaemon(object):
     def test_constructor_calls_prepare(self):
         prepare_daemon_patcher = mock.patch.object(
-            hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
+            http_hooks_deamon.ThreadedHookCallbackDaemon, '_prepare')
         with prepare_daemon_patcher as prepare_daemon_mock:
-            hooks_daemon.ThreadedHookCallbackDaemon()
+            http_hooks_deamon.ThreadedHookCallbackDaemon()
         prepare_daemon_mock.assert_called_once_with()
 
     def test_run_is_called_on_context_start(self):
         patchers = mock.patch.multiple(
-            hooks_daemon.ThreadedHookCallbackDaemon,
+            http_hooks_deamon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
 
         with patchers as mocks:
-            daemon = hooks_daemon.ThreadedHookCallbackDaemon()
+            daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 pass
             mocks['_run'].assert_called_once_with()
         assert daemon_context == daemon
 
     def test_stop_is_called_on_context_exit(self):
         patchers = mock.patch.multiple(
-            hooks_daemon.ThreadedHookCallbackDaemon,
+            http_hooks_deamon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
 
         with patchers as mocks:
-            daemon = hooks_daemon.ThreadedHookCallbackDaemon()
+            daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 assert mocks['_stop'].call_count == 0
 
             mocks['_stop'].assert_called_once_with()
         assert daemon_context == daemon
 
 
 class TestHttpHooksCallbackDaemon(object):
     def test_hooks_callback_generates_new_port(self, caplog):
         with caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon.server_address == ('127.0.0.1', 8881)
 
         with caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host=None, port=None)
         assert daemon._daemon.server_address[1] in range(0, 66000)
         assert daemon._daemon.server_address[0] != '127.0.0.1'
 
     def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon == tcp_server
 
         _, port = tcp_server.server_address
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
-              f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
+              f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
         assert_message_in_log(
-            caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
+            caplog.records, msg, levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_prepare_inits_hooks_uri_and_logs_it(
             self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
 
         _, port = tcp_server.server_address
         expected_uri = '{}:{}'.format('127.0.0.1', port)
         assert daemon.hooks_uri == expected_uri
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
-              f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
+              f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
+
         assert_message_in_log(
             caplog.records, msg,
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_run_creates_a_thread(self, tcp_server):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread) as thread_mock:
             daemon._run()
 
         thread_mock.assert_called_once_with(
             target=tcp_server.serve_forever,
             kwargs={'poll_interval': daemon.POLL_INTERVAL})
         assert thread.daemon is True
         thread.start.assert_called_once_with()
 
     def test_run_logs(self, tcp_server, caplog):
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
             daemon._run()
 
         assert_message_in_log(
             caplog.records,
             'Running thread-based loop of callback daemon in background',
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
             with daemon:
                 assert daemon._daemon == tcp_server
                 assert daemon._callback_thread == thread
 
         assert daemon._daemon is None
         assert daemon._callback_thread is None
         tcp_server.shutdown.assert_called_with()
         thread.join.assert_called_once_with()
 
         assert_message_in_log(
             caplog.records, 'Waiting for background thread to finish.',
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def _tcp_patcher(self, tcp_server):
         return mock.patch.object(
-            hooks_daemon, 'TCPServer', return_value=tcp_server)
+            http_hooks_deamon, 'TCPServer', return_value=tcp_server)
 
     def _thread_patcher(self, thread):
         return mock.patch.object(
-            hooks_daemon.threading, 'Thread', return_value=thread)
+            http_hooks_deamon.threading, 'Thread', return_value=thread)
 
 
 class TestPrepareHooksDaemon(object):
+
     @pytest.mark.parametrize('protocol', ('celery',))
     def test_returns_celery_hooks_callback_daemon_when_celery_protocol_specified(
             self, protocol):
         with tempfile.NamedTemporaryFile(mode='w') as temp_file:
             temp_file.write("[app:main]\ncelery.broker_url = redis://redis/0\n"
                             "celery.result_backend = redis://redis/0")
             temp_file.flush()
             expected_extras = {'config': temp_file.name}
-            callback, extras = hooks_daemon.prepare_callback_daemon(
+            callback, extras = hook_base.prepare_callback_daemon(
                 expected_extras, protocol=protocol, host='')
-            assert isinstance(callback, hooks_daemon.CeleryHooksCallbackDaemon)
+            assert isinstance(callback, celery_hooks_deamon.CeleryHooksCallbackDaemon)
 
     @pytest.mark.parametrize('protocol, expected_class', (
-        ('http', hooks_daemon.HttpHooksCallbackDaemon),
+        ('http', http_hooks_deamon.HttpHooksCallbackDaemon),
     ))
     def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
             self, protocol, expected_class):
         expected_extras = {
             'extra1': 'value1',
             'txn_id': 'txnid2',
             'hooks_protocol': protocol.lower(),
             'task_backend': '',
             'task_queue': ''
         }
-        callback, extras = hooks_daemon.prepare_callback_daemon(
+        callback, extras = hook_base.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
             txn_id='txnid2')
         assert isinstance(callback, expected_class)
         extras.pop('hooks_uri')
         expected_extras['time'] = extras['time']
         assert extras == expected_extras
 
     @pytest.mark.parametrize('protocol', (
         'invalid',
         'Http',
         'HTTP',
     ))
     def test_raises_on_invalid_protocol(self, protocol):
         expected_extras = {
             'extra1': 'value1',
             'hooks_protocol': protocol.lower()
         }
         with pytest.raises(Exception):
-            callback, extras = hooks_daemon.prepare_callback_daemon(
+            callback, extras = hook_base.prepare_callback_daemon(
                 expected_extras.copy(),
                 protocol=protocol, host='127.0.0.1')
 
 
 class MockRequest(object):
 
     def __init__(self, request):
         self.request = request
         self.input_stream = io.BytesIO(safe_bytes(self.request))
         self.output_stream = io.BytesIO()  # make it un-closable for testing invesitagion
         self.output_stream.close = lambda: None
 
     def makefile(self, mode, *args, **kwargs):
         return self.output_stream if mode == 'wb' else self.input_stream
 
 
 class MockServer(object):
 
     def __init__(self, handler_cls, request):
         ip_port = ('0.0.0.0', 8888)
         self.request = MockRequest(request)
         self.server_address = ip_port
         self.handler = handler_cls(self.request, ip_port, self)
 
 
 @pytest.fixture()
 def tcp_server():
     server = mock.Mock()
     server.server_address = ('127.0.0.1', 8881)
     server.wbufsize = 1024
     return server