feat(celery-hooks): added all needed changes to support new celery backend, removed DummyHooksCallbackDaemon, updated tests. Fixes: RCCE-55

rhodecode/apps/ssh_support/lib/backends/__init__.py

@@ -1,262 +1,263 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import datetime
 import configparser
 from sqlalchemy import Table
 
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.model.scm import ScmModel
 
 from .hg import MercurialServer
 from .git import GitServer
 from .svn import SubversionServer
 log = logging.getLogger(__name__)
 
 
 class SshWrapper(object):
     hg_cmd_pat  = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$')
     git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
     svn_cmd_pat = re.compile(r'^svnserve -t')
 
     def __init__(self, command, connection_info, mode,
                  user, user_id, key_id: int, shell, ini_path: str, env):
         self.command = command
         self.connection_info = connection_info
         self.mode = mode
         self.username = user
         self.user_id = user_id
         self.key_id = key_id
         self.shell = shell
         self.ini_path = ini_path
         self.env = env
 
         self.config = self.parse_config(ini_path)
         self.server_impl = None
 
     def parse_config(self, config_path):
         parser = configparser.ConfigParser()
         parser.read(config_path)
         return parser
 
     def update_key_access_time(self, key_id):
         from rhodecode.model.meta import raw_query_executor, Base
 
         table = Table('user_ssh_keys', Base.metadata, autoload=False)
         atime = datetime.datetime.utcnow()
         stmt = (
             table.update()
             .where(table.c.ssh_key_id == key_id)
             .values(accessed_on=atime)
             # no MySQL Support for .returning :((
             #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
         )
 
         res_count = None
         with raw_query_executor() as session:
             result = session.execute(stmt)
             if result.rowcount:
                 res_count = result.rowcount
 
         if res_count:
             log.debug('Update key id:`%s` access time', key_id)
 
     def get_user(self, user_id):
         user = AttributeDict()
         # lazy load db imports
         from rhodecode.model.db import User
         dbuser = User.get(user_id)
         if not dbuser:
             return None
         user.user_id = dbuser.user_id
         user.username = dbuser.username
         user.auth_user = dbuser.AuthUser()
         return user
 
     def get_connection_info(self):
         """
         connection_info
 
         Identifies the client and server ends of the connection.
         The variable contains four space-separated values: client IP address,
         client port number, server IP address, and server port number.
         """
         conn = dict(
             client_ip=None,
             client_port=None,
             server_ip=None,
             server_port=None,
         )
 
         info = self.connection_info.split(' ')
         if len(info) == 4:
             conn['client_ip'] = info[0]
             conn['client_port'] = info[1]
             conn['server_ip'] = info[2]
             conn['server_port'] = info[3]
 
         return conn
 
     def maybe_translate_repo_uid(self, repo_name):
         _org_name = repo_name
         if _org_name.startswith('_'):
             # remove format of _ID/subrepo
             _org_name = _org_name.split('/', 1)[0]
 
         if repo_name.startswith('_'):
             from rhodecode.model.repo import RepoModel
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 repo_name = by_id_match.repo_name
                 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
 
         return repo_name, _org_name
 
     def get_repo_details(self, mode):
         vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
         repo_name = None
 
         hg_match = self.hg_cmd_pat.match(self.command)
         if hg_match is not None:
             vcs_type = 'hg'
             repo_id = hg_match.group(1).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         git_match = self.git_cmd_pat.match(self.command)
         if git_match is not None:
             mode = git_match.group(1)
             vcs_type = 'git'
             repo_id = git_match.group(2).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         svn_match = self.svn_cmd_pat.match(self.command)
         if svn_match is not None:
             vcs_type = 'svn'
             # Repo name should be extracted from the input stream, we're unable to
             # extract it at this point in execution
             return vcs_type, repo_name, mode
 
         return vcs_type, repo_name, mode
 
     def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
         store = ScmModel().repos_path
 
         check_branch_perms = False
         detect_force_push = False
 
         if branch_permissions:
             check_branch_perms = True
             detect_force_push = True
 
         log.debug(
             'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
             vcs, mode, repo, check_branch_perms)
 
         # detect if we have to check branch permissions
         extras = {
             'detect_force_push': detect_force_push,
             'check_branch_perms': check_branch_perms,
+            'config': self.ini_path
         }
 
         if vcs == 'hg':
             server = MercurialServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'git':
             server = GitServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, repo_mode=mode, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'svn':
             server = SubversionServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=None, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         else:
             raise Exception(f'Unrecognised VCS: {vcs}')
 
     def wrap(self):
         mode = self.mode
         username = self.username
         user_id = self.user_id
         key_id = self.key_id
         shell = self.shell
 
         scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
 
         log.debug(
             'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
             'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
             mode, username, user_id, shell, self.command,
             scm_detected, scm_mode, scm_repo)
 
         log.debug('SSH Connection info %s', self.get_connection_info())
 
         # update last access time for this key
         if key_id:
             self.update_key_access_time(key_id)
 
         if shell and self.command is None:
             log.info('Dropping to shell, no command given and shell is allowed')
             os.execl('/bin/bash', '-l')
             exit_code = 1
 
         elif scm_detected:
             user = self.get_user(user_id)
             if not user:
                 log.warning('User with id %s not found', user_id)
                 exit_code = -1
                 return exit_code
 
             auth_user = user.auth_user
             permissions = auth_user.permissions['repositories']
             repo_branch_permissions = auth_user.get_branch_permissions(scm_repo)
             try:
                 exit_code, is_updated = self.serve(
                     scm_detected, scm_repo, scm_mode, user, permissions,
                     repo_branch_permissions)
             except Exception:
                 log.exception('Error occurred during execution of SshWrapper')
                 exit_code = -1
 
         elif self.command is None and shell is False:
             log.error('No Command given.')
             exit_code = -1
 
         else:
             log.error('Unhandled Command: "%s" Aborting.', self.command)
             exit_code = -1
 
         return exit_code

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -1,161 +1,160 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class VcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
     write_perms = ['repository.admin', 'repository.write']
     read_perms = ['repository.read', 'repository.admin', 'repository.write']
 
     def __init__(self, user, user_permissions, config, env):
         self.user = user
         self.user_permissions = user_permissions
         self.config = config
         self.env = env
         self.stdin = sys.stdin
 
         self.repo_name = None
         self.repo_mode = None
         self.store = ''
         self.ini_path = ''
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def has_write_perm(self):
         permission = self.user_permissions.get(self.repo_name)
         if permission in ['repository.write', 'repository.admin']:
             return True
 
         return False
 
     def _check_permissions(self, action):
         permission = self.user_permissions.get(self.repo_name)
         log.debug('permission for %s on %s are: %s',
                   self.user, self.repo_name, permission)
 
         if not permission:
             log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
                       self.user, self.repo_name)
             return -2
 
         if action == 'pull':
             if permission in self.read_perms:
                 log.info(
                     'READ Permissions for User "%s" detected to repo "%s"!',
                     self.user, self.repo_name)
                 return 0
         else:
             if permission in self.write_perms:
                 log.info(
                     'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
                     self.user, self.repo_name)
                 return 0
 
         log.error('Cannot properly fetch or verify user `%s` permissions. '
                   'Permissions: %s, vcs action: %s',
                   self.user, permission, action)
         return -2
 
     def update_environment(self, action, extras=None):
 
         scm_data = {
             'ip': os.environ['SSH_CLIENT'].split()[0],
             'username': self.user.username,
             'user_id': self.user.user_id,
             'action': action,
             'repository': self.repo_name,
             'scm': self.backend,
             'config': self.ini_path,
             'repo_store': self.store,
             'make_lock': None,
             'locked_by': [None, None],
             'server_url': None,
             'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
             'hooks': ['push', 'pull'],
             'hooks_module': 'rhodecode.lib.hooks_daemon',
             'is_shadow_repo': False,
             'detect_force_push': False,
             'check_branch_perms': False,
 
             'SSH': True,
             'SSH_PERMISSIONS': self.user_permissions.get(self.repo_name),
         }
         if extras:
             scm_data.update(extras)
         os.putenv("RC_SCM_DATA", json.dumps(scm_data))
 
     def get_root_store(self):
         root_store = self.store
         if not root_store.endswith('/'):
             # always append trailing slash
             root_store = root_store + '/'
         return root_store
 
     def _handle_tunnel(self, extras):
         # pre-auth
         action = 'pull'
         exit_code = self._check_permissions(action)
         if exit_code:
             return exit_code, False
 
         req = self.env['request']
         server_url = req.host_url + req.script_name
         extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
 
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
-            host=vcs_settings.HOOKS_HOST,
-            use_direct_calls=False)
+            host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             try:
                 return self._handle_tunnel(extras)
             finally:
                 log.debug('Running cleanup with cache invalidation')
                 if self.repo_name:
                     self._invalidate_cache(self.repo_name)

rhodecode/config/utils.py

@@ -1,118 +1,117 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import platform
 
 from rhodecode.model import init_model
 
 
 def configure_vcs(config):
     """
     Patch VCS config with some RhodeCode specific stuff
     """
     from rhodecode.lib.vcs import conf
     import rhodecode.lib.vcs.conf.settings
 
     conf.settings.BACKENDS = {
         'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
         'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
     conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
     conf.settings.HOOKS_HOST = config['vcs.hooks.host']
-    conf.settings.HOOKS_DIRECT_CALLS = config['vcs.hooks.direct_calls']
     conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
     conf.settings.ALIASES[:] = config['vcs.backends']
     conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version']
 
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
 
 def initialize_test_environment(settings, test_env=None):
     if test_env is None:
         test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
 
     from rhodecode.lib.utils import (
         create_test_directory, create_test_database, create_test_repositories,
         create_test_index)
     from rhodecode.tests import TESTS_TMP_PATH
     from rhodecode.lib.vcs.backends.hg import largefiles_store
     from rhodecode.lib.vcs.backends.git import lfs_store
 
     # test repos
     if test_env:
         create_test_directory(TESTS_TMP_PATH)
         # large object stores
         create_test_directory(largefiles_store(TESTS_TMP_PATH))
         create_test_directory(lfs_store(TESTS_TMP_PATH))
 
         create_test_database(TESTS_TMP_PATH, settings)
         create_test_repositories(TESTS_TMP_PATH, settings)
         create_test_index(TESTS_TMP_PATH, settings)
 
 
 def get_vcs_server_protocol(config):
     return config['vcs.server.protocol']
 
 
 def set_instance_id(config):
     """
     Sets a dynamic generated config['instance_id'] if missing or '*'
     E.g instance_id = *cluster-1 or instance_id = *
     """
 
     config['instance_id'] = config.get('instance_id') or ''
     instance_id = config['instance_id']
     if instance_id.startswith('*') or not instance_id:
         prefix = instance_id.lstrip('*')
         _platform_id = platform.uname()[1] or 'instance'
         config['instance_id'] = '{prefix}uname:{platform}-pid:{pid}'.format(
             prefix=prefix,
             platform=_platform_id,
             pid=os.getpid())
 
 
 def get_default_user_id():
     DEFAULT_USER = 'default'
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
         result = session.execute(text("SELECT user_id from users where username = :uname"), {'uname': DEFAULT_USER})
         user_id = result.first()[0]
 
     return user_id
 
 
 def get_default_base_path():
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
         result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'"))
         base_path = result.first()[0]
 
     return base_path

rhodecode/lib/celerylib/tasks.py

@@ -1,412 +1,448 b''
 # Copyright (C) 2012-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
 from pyramid_mailer.mailer import Mailer
 from pyramid_mailer.message import Message
 from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
+from rhodecode.lib.utils import adopt_for_celery
 from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model.db import (
     true, null, Session, IntegrityError, Repository, RepoGroup, User)
 from rhodecode.model.permission import PermissionModel
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None,
                extra_headers=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     :param email_config: specify custom configuration for mailer
     :param extra_headers: specify custom headers
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
 
     subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
 
     if recipients:
         if isinstance(recipients, str):
             recipients = recipients.split(',')
     else:
         # if recipients are not defined we send to email_config + all admins
         admins = []
         for u in User.query().filter(User.admin == true()).all():
             if u.email:
                 admins.append(u.email)
         recipients = []
         config_email = email_config.get('email_to')
         if config_email:
             recipients += [config_email]
         recipients += admins
 
     # translate our LEGACY config into the one that pyramid_mailer supports
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
         username=email_config.get('smtp_username'),
         password=email_config.get('smtp_password'),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
 
         # SSL key file
         # keyfile='',
 
         # SSL certificate file
         # certfile='',
 
         # Location of maildir
         # queue_path='',
 
         default_sender=email_config.get('app_email_from', '[e-mail]'),
 
         debug=str2bool(email_config.get('smtp_debug')),
         # /usr/sbin/sendmail	Sendmail executable
         # sendmail_app='',
 
         # {sendmail_app} -t -i -f {sender}	Template for sendmail execution
         # sendmail_template='',
     )
 
     if extra_headers is None:
         extra_headers = {}
 
     extra_headers.setdefault('Date', formatdate(time.time()))
 
     if 'thread_ids' in extra_headers:
         thread_ids = extra_headers.pop('thread_ids')
         extra_headers['References'] = ' '.join('<{}>'.format(t) for t in thread_ids)
 
     try:
         mailer = Mailer(**email_conf)
 
         message = Message(subject=subject,
                           sender=email_conf['default_sender'],
                           recipients=recipients,
                           body=body, html=html_body,
                           extra_headers=extra_headers)
         mailer.send_immediately(message)
         statsd = StatsdClient.statsd
         if statsd:
             statsd.incr('rhodecode_email_sent_total')
 
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     # set landing rev based on default branches for SCM
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
 
     try:
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         repo.update_commit_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
 
         PermissionModel().trigger_permission_flush()
 
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_ref = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
         repo.update_commit_cache(config=config)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def repo_maintenance(repoid):
     from rhodecode.lib import repo_maintenance as repo_maintenance_lib
     log = get_logger(repo_maintenance)
     repo = Repository.get_by_id_or_repo_name(repoid)
     if repo:
         maintenance = repo_maintenance_lib.RepoMaintenance()
         tasks = maintenance.get_tasks_for_repo(repo)
         log.debug('Executing %s tasks on repo `%s`', tasks, repoid)
         executed_types = maintenance.execute(repo)
         log.debug('Got execution results %s', executed_types)
     else:
         log.debug('Repo `%s` not found or without a clone_url', repoid)
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
 
         current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
         latest_ver = data['versions'][0]['version']
         UpdateModel().store_version(latest_ver)
 
         if send_email_notification:
             log.debug('Send email notification is enabled. '
                       'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
             if UpdateModel().is_outdated(current_ver, latest_ver):
 
                 email_kwargs = {
                     'current_ver': current_ver,
                     'latest_ver': latest_ver,
                 }
 
                 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                     EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
 
                 email_recipients = aslist(email_recipients, sep=',') or \
                                    [user.email for user in User.get_all_super_admins()]
                 run_task(send_email, email_recipients, subject,
                          email_body_plaintext, email_body)
 
     except Exception:
         log.exception('Failed to check for update')
         raise
 
 
 def sync_last_update_for_objects(*args, **kwargs):
     skip_repos = kwargs.get('skip_repos')
     if not skip_repos:
         repos = Repository.query() \
             .order_by(Repository.group_id.asc())
 
         for repo in repos:
             repo.update_commit_cache()
 
     skip_groups = kwargs.get('skip_groups')
     if not skip_groups:
         repo_groups = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == null())
 
         for root_gr in repo_groups:
             for repo_gr in reversed(root_gr.recursive_groups()):
                 repo_gr.update_commit_cache()
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def sync_last_update(*args, **kwargs):
     sync_last_update_for_objects(*args, **kwargs)
 
 
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('%r: Got args: %r and kwargs %r', beat_check, args, kwargs)
     return time.time()
+
+
+@async_task
+@adopt_for_celery
+def repo_size(extras):
+    from rhodecode.lib.hooks_base import repo_size
+    return repo_size(extras)
+
+
+@async_task
+@adopt_for_celery
+def pre_pull(extras):
+    from rhodecode.lib.hooks_base import pre_pull
+    return pre_pull(extras)
+
+
+@async_task
+@adopt_for_celery
+def post_pull(extras):
+    from rhodecode.lib.hooks_base import post_pull
+    return post_pull(extras)
+
+
+@async_task
+@adopt_for_celery
+def pre_push(extras):
+    from rhodecode.lib.hooks_base import pre_push
+    return pre_push(extras)
+
+
+@async_task
+@adopt_for_celery
+def post_push(extras):
+    from rhodecode.lib.hooks_base import post_push
+    return post_push(extras)

rhodecode/lib/hooks_base.py

@@ -1,531 +1,534 b''
 # Copyright (C) 2013-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
+    def to_json(self):
+        return {'status': self.status, 'output': self.output}
+
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = f'.{repo.repo_type}'
     size_vcs, size_root, size_total = _get_scm_size(vcs_part, repo.repo_full_path)
     msg = (f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n')
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
 
         if extras.commit_ids and extras.check_branch_perms:
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
                                    f'FORCE PUSH FORBIDDEN.'
                 else:
                     halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_pull_total', tags=[
             f'user-agent:{user_agent_normalizer(extras.user_agent)}',
         ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `{}`'.format(extras.repository)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_push_total', tags=[
             f'user-agent:{user_agent_normalizer(extras.user_agent)}',
         ])
 
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = f'Released lock on repo `{extras.repository}`\n'
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             pr_link = tmpl.format(ref_type='branch', ref_name=safe_str(branch_name))
             output += f'RhodeCode: open pull request link: {pr_link}\n'
 
         for book_name in extras.new_refs['bookmarks']:
             pr_link = tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name))
             output += f'RhodeCode: open pull request link: {pr_link}\n'
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = f'Repository `{repo_name}` locked by user `{user_name}`. Reason:`{reason}`'
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/hooks_daemon.py

@@ -1,436 +1,451 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import logging
 import tempfile
 import traceback
 import threading
 import socket
 import msgpack
 import gevent
 
 from http.server import BaseHTTPRequestHandler
 from socketserver import TCPServer
 
 import rhodecode
 from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
 from rhodecode.model import meta
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.pyramid_utils import get_config
 from rhodecode.lib.ext_json import json
 from rhodecode.lib import rc_cache
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpHandler(BaseHTTPRequestHandler):
 
     JSON_HOOKS_PROTO = 'json.v1'
     MSGPACK_HOOKS_PROTO = 'msgpack.v1'
     # starting with RhodeCode 5.0.0 MsgPack is the default, prior it used json
     DEFAULT_HOOKS_PROTO = MSGPACK_HOOKS_PROTO
 
     @classmethod
     def serialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
         if proto == cls.MSGPACK_HOOKS_PROTO:
             return msgpack.packb(data)
         return json.dumps(data)
 
     @classmethod
     def deserialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
         if proto == cls.MSGPACK_HOOKS_PROTO:
             return msgpack.unpackb(data)
         return json.loads(data)
 
     def do_POST(self):
         hooks_proto, method, extras = self._read_request()
         log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
 
         txn_id = getattr(self.server, 'txn_id', None)
         if txn_id:
             log.debug('Computing TXN_ID based on `%s`:`%s`',
                       extras['repository'], extras['txn_id'])
             computed_txn_id = rc_cache.utils.compute_key_from_params(
                 extras['repository'], extras['txn_id'])
             if txn_id != computed_txn_id:
                 raise Exception(
                     'TXN ID fail: expected {} got {} instead'.format(
                         txn_id, computed_txn_id))
 
         request = getattr(self.server, 'request', None)
         try:
             hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
             result = self._call_hook_method(hooks, method, extras)
 
         except Exception as e:
             exc_tb = traceback.format_exc()
             result = {
                 'exception': e.__class__.__name__,
                 'exception_traceback': exc_tb,
                 'exception_args': e.args
             }
         self._write_response(hooks_proto, result)
 
     def _read_request(self):
         length = int(self.headers['Content-Length'])
         # respect sent headers, fallback to OLD proto for compatability
         hooks_proto = self.headers.get('rc-hooks-protocol') or self.JSON_HOOKS_PROTO
         if hooks_proto == self.MSGPACK_HOOKS_PROTO:
             # support for new vcsserver msgpack based protocol hooks
             body = self.rfile.read(length)
             data = self.deserialize_data(body)
         else:
             body = self.rfile.read(length)
             data = self.deserialize_data(body)
 
         return hooks_proto, data['method'], data['extras']
 
     def _write_response(self, hooks_proto, result):
         self.send_response(200)
         if hooks_proto == self.MSGPACK_HOOKS_PROTO:
             self.send_header("Content-type", "application/msgpack")
             self.end_headers()
             data = self.serialize_data(result)
             self.wfile.write(data)
         else:
             self.send_header("Content-type", "text/json")
             self.end_headers()
             data = self.serialize_data(result)
             self.wfile.write(data)
 
     def _call_hook_method(self, hooks, method, extras):
         try:
             result = getattr(hooks, method)(extras)
         finally:
             meta.Session.remove()
         return result
 
     def log_message(self, format, *args):
         """
         This is an overridden method of BaseHTTPRequestHandler which logs using
         logging library instead of writing directly to stderr.
         """
 
         message = format % args
 
         log.debug(
             "HOOKS: client=%s - - [%s] %s", self.client_address,
             self.log_date_time_string(), message)
 
 
-class DummyHooksCallbackDaemon(object):
-    hooks_uri = ''
-
+class BaseHooksCallbackDaemon:
+    """
+    Basic context manager for actions that don't require some extra
+    """
     def __init__(self):
         self.hooks_module = Hooks.__module__
 
     def __enter__(self):
         log.debug('Running `%s` callback daemon', self.__class__.__name__)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
 
 
+class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
+    """
+    Context manger for achieving a compatibility with celery backend
+    """
+
+    def __init__(self, config):
+        self.task_queue = config.get('app:main', 'celery.broker_url')
+        self.task_backend = config.get('app:main', 'celery.result_backend')
+
+
 class ThreadedHookCallbackDaemon(object):
 
     _callback_thread = None
     _daemon = None
     _done = False
     use_gevent = False
 
     def __init__(self, txn_id=None, host=None, port=None):
         self._prepare(txn_id=txn_id, host=host, port=port)
         if self.use_gevent:
             self._run_func = self._run_gevent
             self._stop_func = self._stop_gevent
         else:
             self._run_func = self._run
             self._stop_func = self._stop
 
     def __enter__(self):
         log.debug('Running `%s` callback daemon', self.__class__.__name__)
         self._run_func()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
         self._stop_func()
 
     def _prepare(self, txn_id=None, host=None, port=None):
         raise NotImplementedError()
 
     def _run(self):
         raise NotImplementedError()
 
     def _stop(self):
         raise NotImplementedError()
 
     def _run_gevent(self):
         raise NotImplementedError()
 
     def _stop_gevent(self):
         raise NotImplementedError()
 
 
 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     # From Python docs: Polling reduces our responsiveness to a shutdown
     # request and wastes cpu at all other times.
     POLL_INTERVAL = 0.01
 
     use_gevent = False
 
     @property
     def _hook_prefix(self):
         return 'HOOKS: {} '.format(self.hooks_uri)
 
     def get_hostname(self):
         return socket.gethostname() or '127.0.0.1'
 
     def get_available_port(self, min_port=20000, max_port=65535):
         from rhodecode.lib.utils2 import get_available_port as _get_port
         return _get_port(min_port, max_port)
 
     def _prepare(self, txn_id=None, host=None, port=None):
         from pyramid.threadlocal import get_current_request
 
         if not host or host == "*":
             host = self.get_hostname()
         if not port:
             port = self.get_available_port()
 
         server_address = (host, port)
         self.hooks_uri = '{}:{}'.format(host, port)
         self.txn_id = txn_id
         self._done = False
 
         log.debug(
             "%s Preparing HTTP callback daemon registering hook object: %s",
             self._hook_prefix, HooksHttpHandler)
 
         self._daemon = TCPServer(server_address, HooksHttpHandler)
         # inject transaction_id for later verification
         self._daemon.txn_id = self.txn_id
 
         # pass the WEB app request into daemon
         self._daemon.request = get_current_request()
 
     def _run(self):
         log.debug("Running thread-based loop of callback daemon in background")
         callback_thread = threading.Thread(
             target=self._daemon.serve_forever,
             kwargs={'poll_interval': self.POLL_INTERVAL})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _run_gevent(self):
         log.debug("Running gevent-based loop of callback daemon in background")
         # create a new greenlet for the daemon's serve_forever method
         callback_greenlet = gevent.spawn(
             self._daemon.serve_forever,
             poll_interval=self.POLL_INTERVAL)
 
         # store reference to greenlet
         self._callback_greenlet = callback_greenlet
 
         # switch to this greenlet
         gevent.sleep(0.01)
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._daemon.shutdown()
         self._callback_thread.join()
         self._daemon = None
         self._callback_thread = None
         if self.txn_id:
             txn_id_file = get_txn_id_data_path(self.txn_id)
             log.debug('Cleaning up TXN ID %s', txn_id_file)
             if os.path.isfile(txn_id_file):
                 os.remove(txn_id_file)
 
         log.debug("Background thread done.")
 
     def _stop_gevent(self):
         log.debug("Waiting for background greenlet to finish.")
 
         # if greenlet exists and is running
         if self._callback_greenlet and not self._callback_greenlet.dead:
             # shutdown daemon if it exists
             if self._daemon:
                 self._daemon.shutdown()
 
             # kill the greenlet
             self._callback_greenlet.kill()
 
         self._daemon = None
         self._callback_greenlet = None
 
         if self.txn_id:
             txn_id_file = get_txn_id_data_path(self.txn_id)
             log.debug('Cleaning up TXN ID %s', txn_id_file)
             if os.path.isfile(txn_id_file):
                 os.remove(txn_id_file)
 
         log.debug("Background greenlet done.")
 
 
 def get_txn_id_data_path(txn_id):
     import rhodecode
 
     root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
     final_dir = os.path.join(root, 'svn_txn_id')
 
     if not os.path.isdir(final_dir):
         os.makedirs(final_dir)
     return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
 
 
 def store_txn_id_data(txn_id, data_dict):
     if not txn_id:
         log.warning('Cannot store txn_id because it is empty')
         return
 
     path = get_txn_id_data_path(txn_id)
     try:
         with open(path, 'wb') as f:
             f.write(json.dumps(data_dict))
     except Exception:
         log.exception('Failed to write txn_id metadata')
 
 
 def get_txn_id_from_store(txn_id):
     """
     Reads txn_id from store and if present returns the data for callback manager
     """
     path = get_txn_id_data_path(txn_id)
     try:
         with open(path, 'rb') as f:
             return json.loads(f.read())
     except Exception:
         return {}
 
 
-def prepare_callback_daemon(extras, protocol, host, use_direct_calls, txn_id=None):
+def prepare_callback_daemon(extras, protocol, host, txn_id=None):
     txn_details = get_txn_id_from_store(txn_id)
     port = txn_details.get('port', 0)
-    if use_direct_calls:
-        callback_daemon = DummyHooksCallbackDaemon()
-        extras['hooks_module'] = callback_daemon.hooks_module
-    else:
-        if protocol == 'http':
+    match protocol:
+        case 'http':
             callback_daemon = HttpHooksCallbackDaemon(
                 txn_id=txn_id, host=host, port=port)
-        else:
+        case 'celery':
+            callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
+        case 'local':
+            callback_daemon = BaseHooksCallbackDaemon()
+        case _:
             log.error('Unsupported callback daemon protocol "%s"', protocol)
             raise Exception('Unsupported callback daemon protocol.')
 
-    extras['hooks_uri'] = callback_daemon.hooks_uri
+    extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
+    extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
+    extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
     extras['hooks_protocol'] = protocol
     extras['time'] = time.time()
 
     # register txn_id
     extras['txn_id'] = txn_id
-    log.debug('Prepared a callback daemon: %s at url `%s`',
-              callback_daemon.__class__.__name__, callback_daemon.hooks_uri)
+    log.debug('Prepared a callback daemon: %s',
+              callback_daemon.__class__.__name__)
     return callback_daemon, extras
 
 
 class Hooks(object):
     """
     Exposes the hooks for remote call backs
     """
     def __init__(self, request=None, log_prefix=''):
         self.log_prefix = log_prefix
         self.request = request
 
     def repo_size(self, extras):
         log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.repo_size, extras)
 
     def pre_pull(self, extras):
         log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_pull, extras)
 
     def post_pull(self, extras):
         log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_pull, extras)
 
     def pre_push(self, extras):
         log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_push, extras)
 
     def post_push(self, extras):
         log.debug("%sCalled post_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_push, extras)
 
     def _call_hook(self, hook, extras):
         extras = AttributeDict(extras)
         server_url = extras['server_url']
 
         extras.request = self.request
 
         try:
             result = hook(extras)
             if result is None:
                 raise Exception(
                     'Failed to obtain hook result from func: {}'.format(hook))
         except HTTPBranchProtected as handled_error:
             # Those special cases doesn't need error reporting. It's a case of
             # locked repo or protected branch
             result = AttributeDict({
                 'status': handled_error.code,
                 'output': handled_error.explanation
             })
         except (HTTPLockedRC, Exception) as error:
             # locked needs different handling since we need to also
             # handle PULL operations
             exc_tb = ''
             if not isinstance(error, HTTPLockedRC):
                 exc_tb = traceback.format_exc()
                 log.exception('%sException when handling hook %s', self.log_prefix, hook)
             error_args = error.args
             return {
                 'status': 128,
                 'output': '',
                 'exception': type(error).__name__,
                 'exception_traceback': exc_tb,
                 'exception_args': error_args,
             }
         finally:
             meta.Session.remove()
 
         log.debug('%sGot hook call response %s', self.log_prefix, result)
         return {
             'status': result.status,
             'output': result.output,
         }
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass

rhodecode/lib/middleware/simplevcs.py

@@ -1,701 +1,701 b''
 
 
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import io
 import logging
 import importlib
 from functools import wraps
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data: bytes):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
 
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = next(gen)
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'   # repo groups
         '(?P<target>{slug_pat})/'        # target repo
         'pull-request/(?P<pr_id>\\d+)/'  # pull request
         'repository$'                    # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
         realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
 
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_str('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         # Because PEP-3333-WSGI uses bytes-tunneled-in-latin-1 as PATH_INFO
         # and we use this data
         maybe_new_path = '/'.join(data)
         return safe_bytes(maybe_new_path).decode('latin1')
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access repo: `%s` '
                               'repository as anonymous user', self.acl_repo_name)
 
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container|headers based authentication')
 
                 # headers auth, by just reading special headers and bypass the auth with user/passwd
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
 
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got `%s` as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
 
                 # regular auth chain
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = str(user_id)
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
 
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
 
             stream = environ['wsgi.input']
 
             if isinstance(stream, io.BytesIO):
                 data: bytes = stream.getvalue()
             elif hasattr(stream, 'buf'):  # most likely gunicorn.http.body.Body
                 data: bytes = stream.buf.getvalue()
             else:
                 # fallback to the crudest way, copy the iterator
                 data = safe_bytes(stream.read())
                 environ['wsgi.input'] = io.BytesIO(data)
 
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
-        direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
+        protocol = vcs_settings.HOOKS_PROTOCOL
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
-            direct_calls = True
+            protocol = 'local'
 
         return prepare_callback_daemon(
-            extras, protocol=vcs_settings.HOOKS_PROTOCOL,
-            host=vcs_settings.HOOKS_HOST, use_direct_calls=direct_calls, txn_id=txn_id)
+            extras, protocol=protocol,
+            host=vcs_settings.HOOKS_HOST, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/lib/utils.py

@@ -1,808 +1,823 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
 import warnings
+from functools import wraps
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
 
 from mako import exceptions
 
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
+from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
+def adopt_for_celery(func):
+    """
+    Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
+    for further usage as a celery tasks.
+    """
+    @wraps(func)
+    def wrapper(extras):
+        extras = AttributeDict(extras)
+        # HooksResponse implements to_json method which must be used there.
+        return func(extras).to_json()
+    return wrapper
+
+
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     replacement_char = '-'
 
     slug = strip_tags(value)
     slug = convert_accented_entities(slug)
     slug = convert_misc_entities(slug)
 
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub(r'[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
 
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo_name'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     yield from _get_repos(rec_path)
 
     return _get_repos(path)
 
 
 def get_dirpaths(p: str) -> list:
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 "Ignoring path %s since it cannot be decoded into str.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if a given path is a repository group, False otherwise
 
     :param repo_group_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check; we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise OSError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
     for setting in ui_settings:
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in list(app_settings.items()):
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
     return rhodecode.CONFIG['default_base_path']
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in list(initial_repo_list.items()):
         group = map_groups(name)
         str_name = safe_str(name)
         db_repo = repo_model.get_by_repo_name(str_name)
 
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info(force=True)
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks(force=force_hooks_rebuild)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in list(initial_repo_list.keys()):
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    list(initial_repo_list.keys()) if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         def nested_sort(gr):
             return len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     try:
         import rc_testdata
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.lib.utils2 import get_encryption_key
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
 
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return sha256_safe(':'.join(uuid_list))
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)

rhodecode/lib/vcs/conf/settings.py

@@ -1,74 +1,73 b''
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Internal settings for vcs-lib
 """
 
 # list of default encoding used in safe_str methods
 DEFAULT_ENCODINGS = ['utf8']
 
 
 # Compatibility version when creating SVN repositories. None means newest.
 # Other available options are: pre-1.4-compatible, pre-1.5-compatible,
 # pre-1.6-compatible, pre-1.8-compatible
 SVN_COMPATIBLE_VERSION = None
 
 ALIASES = ['hg', 'git', 'svn']
 
 BACKENDS = {
     'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
     'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
     'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
 }
 
 
 ARCHIVE_SPECS = [
     ('tbz2', 'application/x-bzip2', '.tbz2'),
     ('tbz2', 'application/x-bzip2', '.tar.bz2'),
 
     ('tgz', 'application/x-gzip', '.tgz'),
     ('tgz', 'application/x-gzip', '.tar.gz'),
 
     ('zip', 'application/zip', '.zip'),
 ]
 
 HOOKS_PROTOCOL = None
-HOOKS_DIRECT_CALLS = False
 HOOKS_HOST = '127.0.0.1'
 
 
 MERGE_MESSAGE_TMPL = (
     'Merge pull request !{pr_id} from {source_repo} {source_ref_name}\n\n '
     '{pr_title}')
 MERGE_DRY_RUN_MESSAGE = 'dry_run_merge_message_from_rhodecode'
 MERGE_DRY_RUN_USER = 'Dry-Run User'
 MERGE_DRY_RUN_EMAIL = '[e-mail]'
 
 
 def available_aliases():
     """
     Mercurial is required for the system to work, so in case vcs.backends does
     not include it, we make sure it will be available internally
     TODO: anderson: refactor vcs.backends so it won't be necessary, VCS server
     should be responsible to dictate available backends.
     """
     aliases = ALIASES[:]
     if 'hg' not in aliases:
         aliases += ['hg']
     return aliases

rhodecode/model/pull_request.py

@@ -1,2390 +1,2389 b''
 # Copyright (C) 2012-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 pull request model for RhodeCode
 """
 
 import logging
 import os
 
 import datetime
 import urllib.request
 import urllib.parse
 import urllib.error
 import collections
 
 import dataclasses as dataclasses
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.translation import lazy_ugettext
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
 from rhodecode.lib.hash_utils import md5_safe
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import AttributeDict, get_current_rhodecode_user
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason,
     TargetRefMissing, SourceRefMissing)
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError)
 from rhodecode.model import BaseModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     aliased, null, lazyload, and_, or_, select, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule, User)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
     EmailNotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 # Data structure to hold the response data when updating commits during a pull
 # request update.
 class UpdateResponse(object):
 
     def __init__(self, executed, reason, new, old, common_ancestor_id,
                  commit_changes, source_changed, target_changed):
 
         self.executed = executed
         self.reason = reason
         self.new = new
         self.old = old
         self.common_ancestor_id = common_ancestor_id
         self.changes = commit_changes
         self.source_changed = source_changed
         self.target_changed = target_changed
 
 
 def get_diff_info(
         source_repo, source_ref, target_repo, target_ref, get_authors=False,
         get_commit_authors=True):
     """
     Calculates detailed diff information for usage in preview of creation of a pull-request.
     This is also used for default reviewers logic
     """
 
     source_scm = source_repo.scm_instance()
     target_scm = target_repo.scm_instance()
 
     ancestor_id = target_scm.get_common_ancestor(target_ref, source_ref, source_scm)
     if not ancestor_id:
         raise ValueError(
             'cannot calculate diff info without a common ancestor. '
             'Make sure both repositories are related, and have a common forking commit.')
 
     # case here is that want a simple diff without incoming commits,
     # previewing what will be merged based only on commits in the source.
     log.debug('Using ancestor %s as source_ref instead of %s',
               ancestor_id, source_ref)
 
     # source of changes now is the common ancestor
     source_commit = source_scm.get_commit(commit_id=ancestor_id)
     # target commit becomes the source ref as it is the last commit
     # for diff generation this logic gives proper diff
     target_commit = source_scm.get_commit(commit_id=source_ref)
 
     vcs_diff = \
         source_scm.get_diff(commit1=source_commit, commit2=target_commit,
                             ignore_whitespace=False, context=3)
 
     diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
                                          diff_limit=0, file_limit=0, show_full_diff=True)
 
     _parsed = diff_processor.prepare()
 
     all_files = []
     all_files_changes = []
     changed_lines = {}
     stats = [0, 0]
     for f in _parsed:
         all_files.append(f['filename'])
         all_files_changes.append({
             'filename': f['filename'],
             'stats': f['stats']
         })
         stats[0] += f['stats']['added']
         stats[1] += f['stats']['deleted']
 
         changed_lines[f['filename']] = []
         if len(f['chunks']) < 2:
             continue
         # first line is "context" information
         for chunks in f['chunks'][1:]:
             for chunk in chunks['lines']:
                 if chunk['action'] not in ('del', 'mod'):
                     continue
                 changed_lines[f['filename']].append(chunk['old_lineno'])
 
     commit_authors = []
     user_counts = {}
     email_counts = {}
     author_counts = {}
     _commit_cache = {}
 
     commits = []
     if get_commit_authors:
         log.debug('Obtaining commit authors from set of commits')
         _compare_data = target_scm.compare(
             target_ref, source_ref, source_scm, merge=True,
             pre_load=["author", "date", "message"]
         )
 
         for commit in _compare_data:
             # NOTE(marcink): we serialize here, so we don't produce more vcsserver calls on data returned
             # at this function which is later called via JSON serialization
             serialized_commit = dict(
                 author=commit.author,
                 date=commit.date,
                 message=commit.message,
                 commit_id=commit.raw_id,
                 raw_id=commit.raw_id
             )
             commits.append(serialized_commit)
             user = User.get_from_cs_author(serialized_commit['author'])
             if user and user not in commit_authors:
                 commit_authors.append(user)
 
     # lines
     if get_authors:
         log.debug('Calculating authors of changed files')
         target_commit = source_repo.get_commit(ancestor_id)
 
         for fname, lines in changed_lines.items():
 
             try:
                 node = target_commit.get_node(fname, pre_load=["is_binary"])
             except Exception:
                 log.exception("Failed to load node with path %s", fname)
                 continue
 
             if not isinstance(node, FileNode):
                 continue
 
             # NOTE(marcink): for binary node we don't do annotation, just use last author
             if node.is_binary:
                 author = node.last_commit.author
                 email = node.last_commit.author_email
 
                 user = User.get_from_cs_author(author)
                 if user:
                     user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                 author_counts[author] = author_counts.get(author, 0) + 1
                 email_counts[email] = email_counts.get(email, 0) + 1
 
                 continue
 
             for annotation in node.annotate:
                 line_no, commit_id, get_commit_func, line_text = annotation
                 if line_no in lines:
                     if commit_id not in _commit_cache:
                         _commit_cache[commit_id] = get_commit_func()
                     commit = _commit_cache[commit_id]
                     author = commit.author
                     email = commit.author_email
                     user = User.get_from_cs_author(author)
                     if user:
                         user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                     author_counts[author] = author_counts.get(author, 0) + 1
                     email_counts[email] = email_counts.get(email, 0) + 1
 
     log.debug('Default reviewers processing finished')
 
     return {
         'commits': commits,
         'files': all_files_changes,
         'stats': stats,
         'ancestor': ancestor_id,
         # original authors of modified files
         'original_authors': {
             'users': user_counts,
             'authors': author_counts,
             'emails': email_counts,
         },
         'commit_authors': commit_authors
     }
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     DIFF_CONTEXT = diffs.DEFAULT_CONTEXT
 
     UPDATE_STATUS_MESSAGES = {
         UpdateFailureReason.NONE: lazy_ugettext(
             'Pull request update successful.'),
         UpdateFailureReason.UNKNOWN: lazy_ugettext(
             'Pull request update failed because of an unknown error.'),
         UpdateFailureReason.NO_CHANGE: lazy_ugettext(
             'No update needed because the source and target have not changed.'),
         UpdateFailureReason.WRONG_REF_TYPE: lazy_ugettext(
             'Pull request cannot be updated because the reference type is '
             'not supported for an update. Only Branch, Tag or Bookmark is allowed.'),
         UpdateFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be updated because the target '
             'reference is missing.'),
         UpdateFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
     REF_TYPES = ['bookmark', 'book', 'tag', 'branch']
     UPDATABLE_REF_TYPES = ['bookmark', 'book', 'branch']
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
             PullRequest, PullRequestVersion), pull_request)
 
     def _check_perms(self, perms, pull_request, user, api=False):
         if not api:
             return h.HasRepoPermissionAny(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
         else:
             return h.HasRepoPermissionAnyApi(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
 
     def check_user_read(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_merge(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'hg.admin',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_update(self, pull_request, user, api=False):
         owner = user.user_id == pull_request.user_id
         return self.check_user_merge(pull_request, user, api) or owner
 
     def check_user_delete(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         _perms = ('repository.admin',)
         return self._check_perms(_perms, pull_request, user) or owner
 
     def is_user_reviewer(self, pull_request, user):
         return user.user_id in [
             x.user_id for x in
             pull_request.get_pull_request_reviewers(PullRequestReviewers.ROLE_REVIEWER)
             if x.user
         ]
 
     def check_user_change_status(self, pull_request, user, api=False):
         return self.check_user_update(pull_request, user, api) \
                or self.is_user_reviewer(pull_request, user)
 
     def check_user_comment(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         return self.check_user_read(pull_request, user) or owner
 
     def get(self, pull_request):
         return self.__get_pull_request(pull_request)
 
     def _prepare_get_all_query(self, repo_name, search_q=None, source=False,
                                statuses=None, opened_by=None, order_by=None,
                                order_dir='desc', only_created=False):
         repo = None
         if repo_name:
             repo = self._get_repo(repo_name)
 
         q = PullRequest.query()
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         # source or target
         if repo and source:
             q = q.filter(PullRequest.source_repo == repo)
         elif repo:
             q = q.filter(PullRequest.target_repo == repo)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         # opened by filter
         if opened_by:
             q = q.filter(PullRequest.user_id.in_(opened_by))
 
         # only get those that are in "created" state
         if only_created:
             q = q.filter(PullRequest.pull_request_state == PullRequest.STATE_CREATED)
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'id': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_all(self, repo_name, search_q=None, source=False, statuses=None,
                   opened_by=None):
         """
         Count the number of pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by)
 
         return q.count()
 
     def get_all(self, repo_name, search_q=None, source=False, statuses=None,
                 opened_by=None, offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by, order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def count_awaiting_review(self, repo_name, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_review(
             repo_name, search_q=search_q, statuses=statuses)
 
         return len(pull_requests)
 
     def get_awaiting_review(self, repo_name, search_q=None, statuses=None,
                             offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         _filtered_pull_requests = []
         for pr in pull_requests:
             status = pr.calculated_review_status()
             if status in [ChangesetStatus.STATUS_NOT_REVIEWED,
                           ChangesetStatus.STATUS_UNDER_REVIEW]:
                 _filtered_pull_requests.append(pr)
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def _prepare_awaiting_my_review_review_query(
             self, repo_name, user_id, search_q=None, statuses=None,
             order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
         repo_alias = aliased(Repository)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .join(repo_alias,
                   repo_alias.repo_id == pull_request_alias.target_repo_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(repo_alias.repo_name == repo_name) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses)
         return q.count()
 
     def get_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None,
                                offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
 
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_im_participating_query(self, user_id=None, statuses=None, query='',
                                         order_by=None, order_dir='desc'):
         """
         return a query of pull-requests user is an creator, or he's added as a reviewer
         """
         q = PullRequest.query()
         if user_id:
 
             base_query = select(PullRequestReviewers)\
                 .where(PullRequestReviewers.user_id == user_id)\
                 .with_only_columns(PullRequestReviewers.pull_request_id)
 
             user_filter = or_(
                 PullRequest.user_id == user_id,
                 PullRequest.pull_request_id.in_(base_query)
             )
             q = PullRequest.query().filter(user_filter)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in(self, user_id=None, statuses=None, query=''):
         q = self._prepare_im_participating_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in(
             self, user_id=None, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that i'm participating in as a reviewer, or i have opened
         """
 
         q = self._prepare_im_participating_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_participating_in_for_review_query(
             self, user_id, statuses=None, query='', order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in_for_review(self, user_id, statuses=None, query=''):
         q = self._prepare_participating_in_for_review_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in_for_review(
             self, user_id, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that needs user approval or rejection
         """
 
         q = self._prepare_participating_in_for_review_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def get_versions(self, pull_request):
         """
         returns version of pull request sorted by ID descending
         """
         return PullRequestVersion.query()\
             .filter(PullRequestVersion.pull_request == pull_request)\
             .order_by(PullRequestVersion.pull_request_version_id.asc())\
             .all()
 
     def get_pr_version(self, pull_request_id, version=None):
         at_version = None
 
         if version and version == 'latest':
             pull_request_ver = PullRequest.get(pull_request_id)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_obj
             at_version = 'latest'
         elif version:
             pull_request_ver = PullRequestVersion.get_or_404(version)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_ver.pull_request
             at_version = pull_request_ver.pull_request_version_id
         else:
             _org_pull_request_obj = pull_request_obj = PullRequest.get_or_404(
                 pull_request_id)
 
         pull_request_display_obj = PullRequest.get_pr_display_object(
             pull_request_obj, _org_pull_request_obj)
 
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
     def pr_commits_versions(self, versions):
         """
         Maps the pull-request commits into all known PR versions. This way we can obtain
         each pr version the commit was introduced in.
         """
         commit_versions = collections.defaultdict(list)
         num_versions = [x.pull_request_version_id for x in versions]
         for ver in versions:
             for commit_id in ver.revisions:
                 ver_idx = ChangesetComment.get_index_from_version(
                     ver.pull_request_version_id, num_versions=num_versions)
                 commit_versions[commit_id].append(ver_idx)
         return commit_versions
 
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, observers, title, description=None,
                common_ancestor_id=None,
                description_renderer=None,
                reviewer_data=None, translator=None, auth_user=None):
         translator = translator or get_current_request().translate
 
         created_by_user = self._get_user(created_by)
         auth_user = auth_user or created_by_user.AuthUser()
         source_repo = self._get_repo(source_repo)
         target_repo = self._get_repo(target_repo)
 
         pull_request = PullRequest()
         pull_request.source_repo = source_repo
         pull_request.source_ref = source_ref
         pull_request.target_repo = target_repo
         pull_request.target_ref = target_ref
         pull_request.revisions = revisions
         pull_request.title = title
         pull_request.description = description
         pull_request.description_renderer = description_renderer
         pull_request.author = created_by_user
         pull_request.reviewer_data = reviewer_data
         pull_request.pull_request_state = pull_request.STATE_CREATING
         pull_request.common_ancestor_id = common_ancestor_id
 
         Session().add(pull_request)
         Session().flush()
 
         reviewer_ids = set()
         # members / reviewers
         for reviewer_object in reviewers:
             user_id, reasons, mandatory, role, rules = reviewer_object
             user = self._get_user(user_id)
 
             # skip duplicates
             if user.user_id in reviewer_ids:
                 continue
 
             reviewer_ids.add(user.user_id)
 
             reviewer = PullRequestReviewers()
             reviewer.user = user
             reviewer.pull_request = pull_request
             reviewer.reasons = reasons
             reviewer.mandatory = mandatory
             reviewer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 review_group = rule.user_group_vote_rule(user_id)
                 # we check if this particular reviewer is member of a voting group
                 if review_group:
                     # NOTE(marcink):
                     # can be that user is member of more but we pick the first same,
                     # same as default reviewers algo
                     review_group = review_group[0]
 
                     rule_data = {
                         'rule_name':
                             rule.review_rule_name,
                         'rule_user_group_entry_id':
                             review_group.repo_review_rule_users_group_id,
                         'rule_user_group_name':
                             review_group.users_group.users_group_name,
                         'rule_user_group_members':
                             [x.user.username for x in review_group.users_group.members],
                         'rule_user_group_members_id':
                             [x.user.user_id for x in review_group.users_group.members],
                     }
                     # e.g {'vote_rule': -1, 'mandatory': True}
                     rule_data.update(review_group.rule_data())
 
                     reviewer.rule_data = rule_data
 
             Session().add(reviewer)
             Session().flush()
 
         for observer_object in observers:
             user_id, reasons, mandatory, role, rules = observer_object
             user = self._get_user(user_id)
 
             # skip duplicates from reviewers
             if user.user_id in reviewer_ids:
                 continue
 
             #reviewer_ids.add(user.user_id)
 
             observer = PullRequestReviewers()
             observer.user = user
             observer.pull_request = pull_request
             observer.reasons = reasons
             observer.mandatory = mandatory
             observer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 # TODO(marcink): do we need this for observers ??
                 pass
 
             Session().add(observer)
             Session().flush()
 
         # Set approval status to "Under Review" for all commits which are
         # part of this pull request.
         ChangesetStatusModel().set_status(
             repo=target_repo,
             status=ChangesetStatus.STATUS_UNDER_REVIEW,
             user=created_by_user,
             pull_request=pull_request
         )
         # we commit early at this point. This has to do with a fact
         # that before queries do some row-locking. And because of that
         # we need to commit and finish transaction before below validate call
         # that for large repos could be long resulting in long row locks
         Session().commit()
 
         # prepare workspace, and run initial merge simulation. Set state during that
         # operation
         pull_request = PullRequest.get(pull_request.pull_request_id)
 
         # set as merging, for merge simulation, and if finished to created so we mark
         # simulation is working fine
         with pull_request.set_state(PullRequest.STATE_MERGING,
                                     final_state=PullRequest.STATE_CREATED) as state_obj:
             MergeCheck.validate(
                 pull_request, auth_user=auth_user, translator=translator)
 
         self.notify_reviewers(pull_request, reviewer_ids, created_by_user)
         self.trigger_pull_request_hook(pull_request, created_by_user, 'create')
 
         creation_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.create', {'data': creation_data},
             auth_user, pull_request)
 
         return pull_request
 
     def trigger_pull_request_hook(self, pull_request, user, action, data=None):
         pull_request = self.__get_pull_request(pull_request)
         target_scm = pull_request.target_repo.scm_instance()
         if action == 'create':
             trigger_hook = hooks_utils.trigger_create_pull_request_hook
         elif action == 'merge':
             trigger_hook = hooks_utils.trigger_merge_pull_request_hook
         elif action == 'close':
             trigger_hook = hooks_utils.trigger_close_pull_request_hook
         elif action == 'review_status_change':
             trigger_hook = hooks_utils.trigger_review_pull_request_hook
         elif action == 'update':
             trigger_hook = hooks_utils.trigger_update_pull_request_hook
         elif action == 'comment':
             trigger_hook = hooks_utils.trigger_comment_pull_request_hook
         elif action == 'comment_edit':
             trigger_hook = hooks_utils.trigger_comment_pull_request_edit_hook
         else:
             return
 
         log.debug('Handling pull_request %s trigger_pull_request_hook with action %s and hook: %s',
                   pull_request, action, trigger_hook)
         trigger_hook(
             username=user.username,
             repo_name=pull_request.target_repo.repo_name,
             repo_type=target_scm.alias,
             pull_request=pull_request,
             data=data)
 
     def _get_commit_ids(self, pull_request):
         """
         Return the commit ids of the merged pull request.
 
         This method is not dealing correctly yet with the lack of autoupdates
         nor with the implicit target updates.
         For example: if a commit in the source repo is already in the target it
         will be reported anyways.
         """
         merge_rev = pull_request.merge_rev
         if merge_rev is None:
             raise ValueError('This pull request was not merged yet')
 
         commit_ids = list(pull_request.revisions)
         if merge_rev not in commit_ids:
             commit_ids.append(merge_rev)
 
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
         repo_type = pull_request.source_repo.repo_type
         log.debug("Merging pull request %s", pull_request)
 
         extras['user_agent'] = '{}/internal-merge'.format(repo_type)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug("Merge was successful, updating the pull request comments.")
             self._comment_and_close_pr(pull_request, user, merge_state)
 
             self._log_audit_action(
                 'repo.pull_request.merge',
                 {'merge_state': merge_state.__dict__},
                 user, pull_request)
 
         else:
             log.warning("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
 
         message = safe_str(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
             pr_id=pull_request.pull_request_id,
             pr_title=pull_request.title,
             pr_desc=pull_request.description,
             source_repo=source_vcs.name,
             source_ref_name=pull_request.source_ref_parts.name,
             target_repo=target_vcs.name,
             target_ref_name=pull_request.target_ref_parts.name,
         )
 
         workspace_id = self._workspace_id(pull_request)
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         user_name = self._user_name_for_merging(pull_request, user)
 
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
-            host=vcs_settings.HOOKS_HOST,
-            use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
+            host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override
             # for a single call.
             target_vcs.config.set(
                 'rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
             merge_state = target_vcs.merge(
                 repo_id, workspace_id, target_ref, source_vcs,
                 pull_request.source_ref_parts,
                 user_name=user_name, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
 
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
         pull_request.merge_rev = merge_state.merge_ref.commit_id
         pull_request.updated_on = datetime.datetime.now()
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
             text=safe_str(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             closing_pr=True
         )
 
         Session().add(pull_request)
         Session().flush()
         # TODO: paris: replace invalidation with less radical solution
         ScmModel().mark_for_invalidation(
             pull_request.target_repo.repo_name)
         self.trigger_pull_request_hook(pull_request, user, 'merge')
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
         return source_ref_type in self.REF_TYPES
 
     def get_flow_commits(self, pull_request):
 
         # source repo
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_id = pull_request.source_ref_parts.commit_id
         source_repo = pull_request.source_repo.scm_instance()
 
         try:
             if source_ref_type in self.REF_TYPES:
                 source_commit = source_repo.get_commit(
                     source_ref_name, reference_obj=pull_request.source_ref_parts)
             else:
                 source_commit = source_repo.get_commit(source_ref_id)
         except CommitDoesNotExistError:
             raise SourceRefMissing()
 
         # target repo
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_id = pull_request.target_ref_parts.commit_id
         target_repo = pull_request.target_repo.scm_instance()
 
         try:
             if target_ref_type in self.REF_TYPES:
                 target_commit = target_repo.get_commit(
                     target_ref_name, reference_obj=pull_request.target_ref_parts)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
         except CommitDoesNotExistError:
             raise TargetRefMissing()
 
         return source_commit, target_commit
 
     def update_commits(self, pull_request, updating_user):
         """
         Get the updated list of commits for the pull request
         and return the new pull request version and the list
         of commits processed by this update action
 
         updating_user is the user_object who triggered the update
         """
         pull_request = self.__get_pull_request(pull_request)
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_id = pull_request.source_ref_parts.commit_id
 
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         if not self.has_valid_update_type(pull_request):
             log.debug("Skipping update of pull request %s due to ref type: %s",
                       pull_request, source_ref_type)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.WRONG_REF_TYPE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         try:
             source_commit, target_commit = self.get_flow_commits(pull_request)
         except SourceRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_SOURCE_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
         except TargetRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         source_changed = source_ref_id != source_commit.raw_id
         target_changed = target_ref_id != target_commit.raw_id
 
         if not (source_changed or target_changed):
             log.debug("Nothing changed in pull request %s", pull_request)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.NO_CHANGE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=target_changed, target_changed=source_changed)
 
         change_in_found = 'target repo' if target_changed else 'source repo'
         log.debug('Updating pull request because of change in %s detected',
                   change_in_found)
 
         # Finally there is a need for an update, in case of source change
         # we create a new version, else just an update
         if source_changed:
             pull_request_version = self._create_version_from_snapshot(pull_request)
             self._link_comments_to_version(pull_request_version)
         else:
             try:
                 ver = pull_request.versions[-1]
             except IndexError:
                 ver = None
 
             pull_request.pull_request_version_id = \
                 ver.pull_request_version_id if ver else None
             pull_request_version = pull_request
 
         source_repo = pull_request.source_repo.scm_instance()
         target_repo = pull_request.target_repo.scm_instance()
 
         # re-compute commit ids
         old_commit_ids = pull_request.revisions
         pre_load = ["author", "date", "message", "branch"]
         commit_ranges = target_repo.compare(
             target_commit.raw_id, source_commit.raw_id, source_repo, merge=True,
             pre_load=pre_load)
 
         target_ref = target_commit.raw_id
         source_ref = source_commit.raw_id
         ancestor_commit_id = target_repo.get_common_ancestor(
             target_ref, source_ref, source_repo)
 
         if not ancestor_commit_id:
             raise ValueError(
                 'cannot calculate diff info without a common ancestor. '
                 'Make sure both repositories are related, and have a common forking commit.')
 
         pull_request.common_ancestor_id = ancestor_commit_id
 
         pull_request.source_ref = f'{source_ref_type}:{source_ref_name}:{source_commit.raw_id}'
         pull_request.target_ref = f'{target_ref_type}:{target_ref_name}:{ancestor_commit_id}'
 
         pull_request.revisions = [
             commit.raw_id for commit in reversed(commit_ranges)]
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         new_commit_ids = pull_request.revisions
 
         old_diff_data, new_diff_data = self._generate_update_diffs(
             pull_request, pull_request_version)
 
         # calculate commit and file changes
         commit_changes = self._calculate_commit_id_changes(
             old_commit_ids, new_commit_ids)
         file_changes = self._calculate_file_changes(
             old_diff_data, new_diff_data)
 
         # set comments as outdated if DIFFS changed
         CommentsModel().outdate_comments(
             pull_request, old_diff_data=old_diff_data,
             new_diff_data=new_diff_data)
 
         valid_commit_changes = (commit_changes.added or commit_changes.removed)
         file_node_changes = (
             file_changes.added or file_changes.modified or file_changes.removed)
         pr_has_changes = valid_commit_changes or file_node_changes
 
         # Add an automatic comment to the pull request, in case
         # anything has changed
         if pr_has_changes:
             update_comment = CommentsModel().create(
                 text=self._render_update_message(ancestor_commit_id, commit_changes, file_changes),
                 repo=pull_request.target_repo,
                 user=pull_request.author,
                 pull_request=pull_request,
                 send_email=False, renderer=DEFAULT_COMMENTS_RENDERER)
 
             # Update status to "Under Review" for added commits
             for commit_id in commit_changes.added:
                 ChangesetStatusModel().set_status(
                     repo=pull_request.source_repo,
                     status=ChangesetStatus.STATUS_UNDER_REVIEW,
                     comment=update_comment,
                     user=pull_request.author,
                     pull_request=pull_request,
                     revision=commit_id)
 
         # initial commit
         Session().commit()
 
         if pr_has_changes:
             # send update email to users
             try:
                 self.notify_users(pull_request=pull_request, updating_user=updating_user,
                                   ancestor_commit_id=ancestor_commit_id,
                                   commit_changes=commit_changes,
                                   file_changes=file_changes)
                 Session().commit()
             except Exception:
                 log.exception('Failed to send email notification to users')
                 Session().rollback()
 
         log.debug(
             'Updated pull request %s, added_ids: %s, common_ids: %s, '
             'removed_ids: %s', pull_request.pull_request_id,
             commit_changes.added, commit_changes.common, commit_changes.removed)
         log.debug(
             'Updated pull request with the following file changes: %s',
             file_changes)
 
         log.info(
             "Updated pull request %s from commit %s to commit %s, "
             "stored new version %s of this pull request.",
             pull_request.pull_request_id, source_ref_id,
             pull_request.source_ref_parts.commit_id,
             pull_request_version.pull_request_version_id)
 
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'update')
 
         return UpdateResponse(
             executed=True, reason=UpdateFailureReason.NONE,
             old=pull_request, new=pull_request_version,
             common_ancestor_id=ancestor_commit_id, commit_changes=commit_changes,
             source_changed=source_changed, target_changed=target_changed)
 
     def _create_version_from_snapshot(self, pull_request):
         version = PullRequestVersion()
         version.title = pull_request.title
         version.description = pull_request.description
         version.status = pull_request.status
         version.pull_request_state = pull_request.pull_request_state
         version.created_on = datetime.datetime.now()
         version.updated_on = pull_request.updated_on
         version.user_id = pull_request.user_id
         version.source_repo = pull_request.source_repo
         version.source_ref = pull_request.source_ref
         version.target_repo = pull_request.target_repo
         version.target_ref = pull_request.target_ref
 
         version._last_merge_source_rev = pull_request._last_merge_source_rev
         version._last_merge_target_rev = pull_request._last_merge_target_rev
         version.last_merge_status = pull_request.last_merge_status
         version.last_merge_metadata = pull_request.last_merge_metadata
         version.shadow_merge_ref = pull_request.shadow_merge_ref
         version.merge_rev = pull_request.merge_rev
         version.reviewer_data = pull_request.reviewer_data
 
         version.revisions = pull_request.revisions
         version.common_ancestor_id = pull_request.common_ancestor_id
         version.pull_request = pull_request
         Session().add(version)
         Session().flush()
 
         return version
 
     def _generate_update_diffs(self, pull_request, pull_request_version):
 
         diff_context = (
             self.DIFF_CONTEXT +
             CommentsModel.needed_extra_diff_context())
         hide_whitespace_changes = False
         source_repo = pull_request_version.source_repo
         source_ref_id = pull_request_version.source_ref_parts.commit_id
         target_ref_id = pull_request_version.target_ref_parts.commit_id
         old_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         new_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         # NOTE: this was using diff_format='gitdiff'
         old_diff_data = diffs.DiffProcessor(old_diff, diff_format='newdiff')
         old_diff_data.prepare()
         new_diff_data = diffs.DiffProcessor(new_diff, diff_format='newdiff')
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
 
     def _link_comments_to_version(self, pull_request_version):
         """
         Link all unlinked comments of this pull request to the given version.
 
         :param pull_request_version: The `PullRequestVersion` to which
             the comments shall be linked.
 
         """
         pull_request = pull_request_version.pull_request
         comments = ChangesetComment.query()\
             .filter(
                 # TODO: johbo: Should we query for the repo at all here?
                 # Pending decision on how comments of PRs are to be related
                 # to either the source repo, the target repo or no repo at all.
                 ChangesetComment.repo_id == pull_request.target_repo.repo_id,
                 ChangesetComment.pull_request == pull_request,
                 ChangesetComment.pull_request_version == null())\
             .order_by(ChangesetComment.comment_id.asc())
 
         # TODO: johbo: Find out why this breaks if it is done in a bulk
         # operation.
         for comment in comments:
             comment.pull_request_version_id = (
                 pull_request_version.pull_request_version_id)
             Session().add(comment)
 
     def _calculate_commit_id_changes(self, old_ids, new_ids):
         added = [x for x in new_ids if x not in old_ids]
         common = [x for x in new_ids if x in old_ids]
         removed = [x for x in old_ids if x not in new_ids]
         total = new_ids
         return ChangeTuple(added, common, removed, total)
 
     def _calculate_file_changes(self, old_diff_data, new_diff_data):
 
         old_files = OrderedDict()
         for diff_data in old_diff_data.parsed_diff:
             old_files[diff_data['filename']] = md5_safe(diff_data['raw_diff'])
 
         added_files = []
         modified_files = []
         removed_files = []
         for diff_data in new_diff_data.parsed_diff:
             new_filename = diff_data['filename']
             new_hash = md5_safe(diff_data['raw_diff'])
 
             old_hash = old_files.get(new_filename)
             if not old_hash:
                 # file is not present in old diff, we have to figure out from parsed diff
                 # operation ADD/REMOVE
                 operations_dict = diff_data['stats']['ops']
                 if diffs.DEL_FILENODE in operations_dict:
                     removed_files.append(new_filename)
                 else:
                     added_files.append(new_filename)
             else:
                 if new_hash != old_hash:
                     modified_files.append(new_filename)
                 # now remove a file from old, since we have seen it already
                 del old_files[new_filename]
 
         # removed files is when there are present in old, but not in NEW,
         # since we remove old files that are present in new diff, left-overs
         # if any should be the removed files
         removed_files.extend(old_files.keys())
 
         return FileChangeTuple(added_files, modified_files, removed_files)
 
     def _render_update_message(self, ancestor_commit_id, changes, file_changes):
         """
         render the message using DEFAULT_COMMENTS_RENDERER (RST renderer),
         so it's always looking the same disregarding on which default
         renderer system is using.
 
         :param ancestor_commit_id: ancestor raw_id
         :param changes: changes named tuple
         :param file_changes: file changes named tuple
 
         """
         new_status = ChangesetStatus.get_status_lbl(
             ChangesetStatus.STATUS_UNDER_REVIEW)
 
         changed_files = (
             file_changes.added + file_changes.modified + file_changes.removed)
 
         params = {
             'under_review_label': new_status,
             'added_commits': changes.added,
             'removed_commits': changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'ancestor_commit_id': ancestor_commit_id
         }
         renderer = RstTemplateRenderer()
         return renderer.render('pull_request_update.mako', **params)
 
     def edit(self, pull_request, title, description, description_renderer, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
         if title:
             pull_request.title = title
         pull_request.description = description
         pull_request.updated_on = datetime.datetime.now()
         pull_request.description_renderer = description_renderer
         Session().add(pull_request)
         self._log_audit_action(
             'repo.pull_request.edit', {'old_data': old_data},
             user, pull_request)
 
     def update_reviewers(self, pull_request, reviewer_data, user):
         """
         Update the reviewers in the pull request
 
         :param pull_request: the pr to update
         :param reviewer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
 
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         reviewers = {}
         for user_id, reasons, mandatory, role, rules in reviewer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             reviewers[user_id] = {
                 'reasons': reasons, 'mandatory': mandatory, 'role': role}
 
         reviewers_ids = set(reviewers.keys())
         current_reviewers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_REVIEWER)
 
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         ids_to_add = reviewers_ids.difference(current_reviewers_ids)
         ids_to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers", ids_to_add)
         log.debug("Removing %s reviewers", ids_to_remove)
         changed = False
         added_audit_reviewers = []
         removed_audit_reviewers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             reviewer = PullRequestReviewers()
             reviewer.user = _usr
             reviewer.pull_request = pull_request
             reviewer.reasons = reviewers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # reviewer.mandatory = reviewers[uid]['reasons']
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             reviewer.role = PullRequestReviewers.ROLE_REVIEWER
             Session().add(reviewer)
             added_audit_reviewers.append(reviewer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             reviewers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_REVIEWER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in reviewers:
                 added_audit_reviewers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_reviewers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def update_observers(self, pull_request, observer_data, user):
         """
         Update the observers in the pull request
 
         :param pull_request: the pr to update
         :param observer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         observers = {}
         for user_id, reasons, mandatory, role, rules in observer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             observers[user_id] = {
                 'reasons': reasons, 'observers': mandatory, 'role': role}
 
         observers_ids = set(observers.keys())
         current_observers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_OBSERVER)
 
         current_observers_ids = set([x.user.user_id for x in current_observers])
 
         ids_to_add = observers_ids.difference(current_observers_ids)
         ids_to_remove = current_observers_ids.difference(observers_ids)
 
         log.debug("Adding %s observer", ids_to_add)
         log.debug("Removing %s observer", ids_to_remove)
         changed = False
         added_audit_observers = []
         removed_audit_observers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             observer = PullRequestReviewers()
             observer.user = _usr
             observer.pull_request = pull_request
             observer.reasons = observers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # observer.mandatory = observer[uid]['reasons']
 
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             observer.role = PullRequestReviewers.ROLE_OBSERVER
             Session().add(observer)
             added_audit_observers.append(observer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             observers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_OBSERVER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in observers:
                 added_audit_observers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_observers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def get_url(self, pull_request, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if permalink:
             return request.route_url(
                 'pull_requests_global',
                 pull_request_id=pull_request.pull_request_id,)
         else:
             return request.route_url('pullrequest_show',
                 repo_name=safe_str(pull_request.target_repo.repo_name),
                 pull_request_id=pull_request.pull_request_id,)
 
     def get_shadow_clone_url(self, pull_request, request=None):
         """
         Returns qualified url pointing to the shadow repository. If this pull
         request is closed there is no shadow repository and ``None`` will be
         returned.
         """
         if pull_request.is_closed():
             return None
         else:
             pr_url = urllib.parse.unquote(self.get_url(pull_request, request=request))
             return safe_str('{pr_url}/repository'.format(pr_url=pr_url))
 
     def _notify_reviewers(self, pull_request, user_ids, role, user):
         # notification to reviewers/observers
         if not user_ids:
             return
 
         log.debug('Notify following %s users about pull-request %s', role, user_ids)
 
         pull_request_obj = pull_request
         # get the current participants of this pull request
         recipients = user_ids
         notification_type = EmailNotificationModel.TYPE_PULL_REQUEST
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         # pull request specifics
         pull_request_commits = [
             (x.raw_id, x.message)
             for x in map(pr_source_repo.get_commit, pull_request.revisions)]
 
         current_rhodecode_user = user
         kwargs = {
             'user': current_rhodecode_user,
             'pull_request_author': pull_request.author,
             'pull_request': pull_request_obj,
             'pull_request_commits': pull_request_commits,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
             'thread_ids': [pr_url],
             'user_role': role
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=current_rhodecode_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=notification_type,
             recipients=recipients,
             email_kwargs=kwargs,
         )
 
     def notify_reviewers(self, pull_request, reviewers_ids, user):
         return self._notify_reviewers(pull_request, reviewers_ids,
                                       PullRequestReviewers.ROLE_REVIEWER, user)
 
     def notify_observers(self, pull_request, observers_ids, user):
         return self._notify_reviewers(pull_request, observers_ids,
                                       PullRequestReviewers.ROLE_OBSERVER, user)
 
     def notify_users(self, pull_request, updating_user, ancestor_commit_id,
                      commit_changes, file_changes):
 
         updating_user_id = updating_user.user_id
         reviewers = set([x.user.user_id for x in pull_request.get_pull_request_reviewers()])
         # NOTE(marcink): send notification to all other users except to
         # person who updated the PR
         recipients = reviewers.difference(set([updating_user_id]))
 
         log.debug('Notify following recipients about pull-request update %s', recipients)
 
         pull_request_obj = pull_request
 
         # send email about the update
         changed_files = (
                 file_changes.added + file_changes.modified + file_changes.removed)
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'updating_user': updating_user,
 
             'pull_request': pull_request_obj,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
 
             'ancestor_commit_id': ancestor_commit_id,
             'added_commits': commit_changes.added,
             'removed_commits': commit_changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'thread_ids': [pr_url],
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=updating_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=EmailNotificationModel.TYPE_PULL_REQUEST_UPDATE,
             recipients=recipients,
             email_kwargs=email_kwargs,
         )
 
     def delete(self, pull_request, user=None):
         if not user:
             user = getattr(get_current_rhodecode_user(), 'username', None)
 
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         self._cleanup_merge_workspace(pull_request)
         self._log_audit_action(
             'repo.pull_request.delete', {'old_data': old_data},
             user, pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         self._cleanup_merge_workspace(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'close')
 
         pr_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
             self, pull_request, user, repo, message=None, auth_user=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
         if pull_request_review_status == ChangesetStatus.STATUS_APPROVED:
             # approved only if we have voting consent
             status = ChangesetStatus.STATUS_APPROVED
         else:
             status = ChangesetStatus.STATUS_REJECTED
         status_lbl = ChangesetStatus.get_status_lbl(status)
 
         default_message = (
             'Closing with status change {transition_icon} {status}.'
         ).format(transition_icon='>', status=status_lbl)
         text = message or default_message
 
         # create a comment, and link it to new status
         comment = CommentsModel().create(
             text=text,
             repo=repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
             closing_pr=True,
             auth_user=auth_user,
         )
 
         # calculate old status before we change it
         old_calculated_status = pull_request.calculated_review_status()
         ChangesetStatusModel().set_status(
             repo.repo_id,
             status,
             user.user_id,
             comment=comment,
             pull_request=pull_request.pull_request_id
         )
 
         Session().flush()
 
         self.trigger_pull_request_hook(pull_request, user, 'comment',
                                        data={'comment': comment})
 
         # we now calculate the status of pull request again, and based on that
         # calculation trigger status change. This might happen in cases
         # that non-reviewer admin closes a pr, which means his vote doesn't
         # change the status, while if he's a reviewer this might change it.
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             self.trigger_pull_request_hook(pull_request, user, 'review_status_change',
                                            data={'status': calculated_status})
 
         # finally close the PR
         PullRequestModel().close_pull_request(pull_request.pull_request_id, user)
 
         return comment, status
 
     def merge_status(self, pull_request, translator=None, force_shadow_repo_refresh=False):
         _ = translator or get_current_request().translate
 
         if not self._is_merge_enabled(pull_request):
             return None, False, _('Server-side pull request merging is disabled.')
 
         if pull_request.is_closed():
             return None, False, _('This pull request is closed.')
 
         merge_possible, msg = self._check_repo_requirements(
             target=pull_request.target_repo, source=pull_request.source_repo,
             translator=_)
         if not merge_possible:
             return None, merge_possible, msg
 
         try:
             merge_response = self._try_merge(
                 pull_request, force_shadow_repo_refresh=force_shadow_repo_refresh)
             log.debug("Merge response: %s", merge_response)
             return merge_response, merge_response.possible, merge_response.merge_status_message
         except NotImplementedError:
             return None, False, _('Pull request merging is not supported.')
 
     def _check_repo_requirements(self, target, source, translator):
         """
         Check if `target` and `source` have compatible requirements.
 
         Currently this is just checking for largefiles.
         """
         _ = translator
         target_has_largefiles = self._has_largefiles(target)
         source_has_largefiles = self._has_largefiles(source)
         merge_possible = True
         message = u''
 
         if target_has_largefiles != source_has_largefiles:
             merge_possible = False
             if source_has_largefiles:
                 message = _(
                     'Target repository large files support is disabled.')
             else:
                 message = _(
                     'Source repository large files support is disabled.')
 
         return merge_possible, message
 
     def _has_largefiles(self, repo):
         largefiles_ui = VcsSettingsModel(repo=repo).get_ui_settings(
             'extensions', 'largefiles')
         return largefiles_ui and largefiles_ui[0].active
 
     def _try_merge(self, pull_request, force_shadow_repo_refresh=False):
         """
         Try to merge the pull request and return the merge status.
         """
         log.debug(
             "Trying out if the pull request %s can be merged. Force_refresh=%s",
             pull_request.pull_request_id, force_shadow_repo_refresh)
         target_vcs = pull_request.target_repo.scm_instance()
         # Refresh the target reference.
         try:
             target_ref = self._refresh_reference(
                 pull_request.target_ref_parts, target_vcs)
         except CommitDoesNotExistError:
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF,
                 metadata={'target_ref': pull_request.target_ref_parts})
             return merge_state
 
         target_locked = pull_request.target_repo.locked
         if target_locked and target_locked[0]:
             locked_by = 'user:{}'.format(target_locked[0])
             log.debug("The target repository is locked by %s.", locked_by)
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_LOCKED,
                 metadata={'locked_by': locked_by})
         elif force_shadow_repo_refresh or self._needs_merge_state_refresh(
                 pull_request, target_ref):
             log.debug("Refreshing the merge status of the repository.")
             merge_state = self._refresh_merge_state(
                 pull_request, target_vcs, target_ref)
         else:
             possible = pull_request.last_merge_status == MergeFailureReason.NONE
             metadata = {
                 'unresolved_files': '',
                 'target_ref': pull_request.target_ref_parts,
                 'source_ref': pull_request.source_ref_parts,
             }
             if pull_request.last_merge_metadata:
                 metadata.update(pull_request.last_merge_metadata_parsed)
 
             if not possible and target_ref.type == 'branch':
                 # NOTE(marcink): case for mercurial multiple heads on branch
                 heads = target_vcs._heads(target_ref.name)
                 if len(heads) != 1:
                     heads = '\n,'.join(target_vcs._heads(target_ref.name))
                     metadata.update({
                         'heads': heads
                     })
 
             merge_state = MergeResponse(
                 possible, False, None, pull_request.last_merge_status, metadata=metadata)
 
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
         if reference.type in self.UPDATABLE_REF_TYPES:
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id
 
         refreshed_commit = vcs_repository.get_commit(name_or_id)
         refreshed_reference = Reference(
             reference.type, reference.name, refreshed_commit.raw_id)
         return refreshed_reference
 
     def _needs_merge_state_refresh(self, pull_request, target_reference):
         return not(
             pull_request.revisions and
             pull_request.revisions[0] == pull_request._last_merge_source_rev and
             target_reference.commit_id == pull_request._last_merge_target_rev)
 
     def _refresh_merge_state(self, pull_request, target_vcs, target_reference):
         workspace_id = self._workspace_id(pull_request)
         source_vcs = pull_request.source_repo.scm_instance()
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         merge_state = target_vcs.merge(
             repo_id, workspace_id,
             target_reference, source_vcs, pull_request.source_ref_parts,
             dry_run=True, use_rebase=use_rebase,
             close_branch=close_branch)
 
         # Do not store the response if there was an unknown error.
         if merge_state.failure_reason != MergeFailureReason.UNKNOWN:
             pull_request._last_merge_source_rev = \
                 pull_request.source_ref_parts.commit_id
             pull_request._last_merge_target_rev = target_reference.commit_id
             pull_request.last_merge_status = merge_state.failure_reason
             pull_request.last_merge_metadata = merge_state.metadata
 
             pull_request.shadow_merge_ref = merge_state.merge_ref
             Session().add(pull_request)
             Session().commit()
 
         return merge_state
 
     def _workspace_id(self, pull_request):
         workspace_id = 'pr-%s' % pull_request.pull_request_id
         return workspace_id
 
     def generate_repo_data(self, repo, commit_id=None, branch=None,
                            bookmark=None, translator=None):
         from rhodecode.model.repo import RepoModel
 
         all_refs, selected_ref = \
             self._get_repo_pullrequest_sources(
                 repo.scm_instance(), commit_id=commit_id,
                 branch=branch, bookmark=bookmark, translator=translator)
 
         refs_select2 = []
         for element in all_refs:
             children = [{'id': x[0], 'text': x[1]} for x in element[0]]
             refs_select2.append({'text': element[1], 'children': children})
 
         return {
             'user': {
                 'user_id': repo.user.user_id,
                 'username': repo.user.username,
                 'firstname': repo.user.first_name,
                 'lastname': repo.user.last_name,
                 'gravatar_link': h.gravatar_url(repo.user.email, 14),
             },
             'name': repo.repo_name,
             'link': RepoModel().get_url(repo),
             'description': h.chop_at_smart(repo.description_safe, '\n'),
             'refs': {
                 'all_refs': all_refs,
                 'selected_ref': selected_ref,
                 'select2_refs': refs_select2
             }
         }
 
     def generate_pullrequest_title(self, source, source_ref, target):
         return u'{source}#{at_ref} to {target}'.format(
             source=source,
             at_ref=source_ref,
             target=target,
         )
 
     def _cleanup_merge_workspace(self, pull_request):
         # Merging related cleanup
         repo_id = pull_request.target_repo.repo_id
         target_scm = pull_request.target_repo.scm_instance()
         workspace_id = self._workspace_id(pull_request)
 
         try:
             target_scm.cleanup_merge_workspace(repo_id, workspace_id)
         except NotImplementedError:
             pass
 
     def _get_repo_pullrequest_sources(
             self, repo, commit_id=None, branch=None, bookmark=None,
             translator=None):
         """
         Return a structure with repo's interesting commits, suitable for
         the selectors in pullrequest controller
 
         :param commit_id: a commit that must be in the list somehow
             and selected by default
         :param branch: a branch that must be in the list and selected
             by default - even if closed
         :param bookmark: a bookmark that must be in the list and selected
         """
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
         branch = safe_str(branch) if branch else None
         bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
         # order matters: first source that has commit_id in it will be selected
         sources = []
         sources.append(('book', repo.bookmarks.items(), _('Bookmarks'), bookmark))
         sources.append(('branch', repo.branches.items(), _('Branches'), branch))
 
         if commit_id:
             ref_commit = (h.short_id(commit_id), commit_id)
             sources.append(('rev', [ref_commit], _('Commit IDs'), commit_id))
 
         sources.append(
             ('branch', repo.branches_closed.items(), _('Closed Branches'), branch),
         )
 
         groups = []
 
         for group_key, ref_list, group_name, match in sources:
             group_refs = []
             for ref_name, ref_id in ref_list:
                 ref_key = u'{}:{}:{}'.format(group_key, ref_name, ref_id)
                 group_refs.append((ref_key, ref_name))
 
                 if not selected:
                     if set([commit_id, match]) & set([ref_id, ref_name]):
                         selected = ref_key
 
             if group_refs:
                 groups.append((group_refs, group_name))
 
         if not selected:
             ref = commit_id or branch or bookmark
             if ref:
                 raise CommitDoesNotExistError(
                     u'No commit refs could be found matching: {}'.format(ref))
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = u'branch:{}:{}'.format(
                     safe_str(repo.DEFAULT_BRANCH_NAME),
                     safe_str(repo.branches[repo.DEFAULT_BRANCH_NAME])
                 )
             elif repo.commit_ids:
                 # make the user select in this case
                 selected = None
             else:
                 raise EmptyRepositoryError()
         return groups, selected
 
     def get_diff(self, source_repo, source_ref_id, target_ref_id,
                  hide_whitespace_changes, diff_context):
 
         return self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
     def _get_diff_from_pr_or_version(
             self, source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context):
 
         target_commit = source_repo.get_commit(
             commit_id=safe_str(target_ref_id))
         source_commit = source_repo.get_commit(
             commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         if isinstance(source_repo, Repository):
             vcs_repo = source_repo.scm_instance()
         else:
             vcs_repo = source_repo
 
         # TODO: johbo: In the context of an update, we cannot reach
         # the old commit anymore with our normal mechanisms. It needs
         # some sort of special support in the vcs layer to avoid this
         # workaround.
         if (source_commit.raw_id == vcs_repo.EMPTY_COMMIT_ID and
                 vcs_repo.alias == 'git'):
             source_commit.raw_id = safe_str(source_ref_id)
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
                   safe_str(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit,
             ignore_whitespace=hide_whitespace_changes, context=diff_context)
         return vcs_diff
 
     def _is_merge_enabled(self, pull_request):
         return self._get_general_setting(
             pull_request, 'rhodecode_pr_merge_enabled')
 
     def _use_rebase_for_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_use_rebase_for_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_use_rebase_for_merging')
 
         return False
 
     def _user_name_for_merging(self, pull_request, user):
         env_user_name_attr = os.environ.get('RC_MERGE_USER_NAME_ATTR', '')
         if env_user_name_attr and hasattr(user, env_user_name_attr):
             user_name_attr = env_user_name_attr
         else:
             user_name_attr = 'short_contact'
 
         user_name = getattr(user, user_name_attr)
         return user_name
 
     def _close_branch_before_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_close_branch_before_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_close_branch_before_merging')
 
         return False
 
     def _get_general_setting(self, pull_request, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=pull_request.target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _log_audit_action(self, action, action_data, user, pull_request):
         audit_logger.store(
             action=action,
             action_data=action_data,
             user=user,
             repo=pull_request.target_repo)
 
     def get_reviewer_functions(self):
         """
         Fetches functions for validation and fetching default reviewers.
         If available we use the EE package, else we fallback to CE
         package functions
         """
         try:
             from rc_reviewers.utils import get_default_reviewers_data
             from rc_reviewers.utils import validate_default_reviewers
             from rc_reviewers.utils import validate_observers
         except ImportError:
             from rhodecode.apps.repository.utils import get_default_reviewers_data
             from rhodecode.apps.repository.utils import validate_default_reviewers
             from rhodecode.apps.repository.utils import validate_observers
 
         return get_default_reviewers_data, validate_default_reviewers, validate_observers
 
 
 class MergeCheck(object):
     """
     Perform Merge Checks and returns a check object which stores information
     about merge errors, and merge conditions
     """
     TODO_CHECK = 'todo'
     PERM_CHECK = 'perm'
     REVIEW_CHECK = 'review'
     MERGE_CHECK = 'merge'
     WIP_CHECK = 'wip'
 
     def __init__(self):
         self.review_status = None
         self.merge_possible = None
         self.merge_msg = ''
         self.merge_response = None
         self.failed = None
         self.errors = []
         self.error_details = OrderedDict()
         self.source_commit = AttributeDict()
         self.target_commit = AttributeDict()
         self.reviewers_count = 0
         self.observers_count = 0
 
     def __repr__(self):
         return '<MergeCheck(possible:{}, failed:{}, errors:{})>'.format(
             self.merge_possible, self.failed, self.errors)
 
     def push_error(self, error_type, message, error_key, details):
         self.failed = True
         self.errors.append([error_type, message])
         self.error_details[error_key] = dict(
             details=details,
             error_type=error_type,
             message=message
         )
 
     @classmethod
     def validate(cls, pull_request, auth_user, translator, fail_early=False,
                  force_shadow_repo_refresh=False):
         _ = translator
         merge_check = cls()
 
         # title has WIP:
         if pull_request.work_in_progress:
             log.debug("MergeCheck: cannot merge, title has wip: marker.")
 
             msg = _('WIP marker in title prevents from accidental merge.')
             merge_check.push_error('error', msg, cls.WIP_CHECK, pull_request.title)
             if fail_early:
                 return merge_check
 
         # permissions to merge
         user_allowed_to_merge = PullRequestModel().check_user_merge(pull_request, auth_user)
         if not user_allowed_to_merge:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('User `{}` not allowed to perform merge.').format(auth_user.username)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # permission to merge into the target branch
         target_commit_id = pull_request.target_ref_parts.commit_id
         if pull_request.target_ref_parts.type == 'branch':
             branch_name = pull_request.target_ref_parts.name
         else:
             # for mercurial we can always figure out the branch from the commit
             # in case of bookmark
             target_commit = pull_request.target_repo.get_commit(target_commit_id)
             branch_name = target_commit.branch
 
         rule, branch_perm = auth_user.get_rule_and_branch_permission(
             pull_request.target_repo.repo_name, branch_name)
         if branch_perm and branch_perm == 'branch.none':
             msg = _('Target branch `{}` changes rejected by rule {}.').format(
                 branch_name, rule)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # review status, must be always present
         review_status = pull_request.calculated_review_status()
         merge_check.review_status = review_status
         merge_check.reviewers_count = pull_request.reviewers_count
         merge_check.observers_count = pull_request.observers_count
 
         status_approved = review_status == ChangesetStatus.STATUS_APPROVED
         if not status_approved and merge_check.reviewers_count:
             log.debug("MergeCheck: cannot merge, approval is pending.")
             msg = _('Pull request reviewer approval is pending.')
 
             merge_check.push_error('warning', msg, cls.REVIEW_CHECK, review_status)
 
             if fail_early:
                 return merge_check
 
         # left over TODOs
         todos = CommentsModel().get_pull_request_unresolved_todos(pull_request)
         if todos:
             log.debug("MergeCheck: cannot merge, {} "
                       "unresolved TODOs left.".format(len(todos)))
 
             if len(todos) == 1:
                 msg = _('Cannot merge, {} TODO still not resolved.').format(
                     len(todos))
             else:
                 msg = _('Cannot merge, {} TODOs still not resolved.').format(
                     len(todos))
 
             merge_check.push_error('warning', msg, cls.TODO_CHECK, todos)
 
             if fail_early:
                 return merge_check
 
         # merge possible, here is the filesystem simulation + shadow repo
         merge_response, merge_status, msg = PullRequestModel().merge_status(
             pull_request, translator=translator,
             force_shadow_repo_refresh=force_shadow_repo_refresh)
 
         merge_check.merge_possible = merge_status
         merge_check.merge_msg = msg
         merge_check.merge_response = merge_response
 
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         try:
             source_commit, target_commit = PullRequestModel().get_flow_commits(pull_request)
             merge_check.source_commit.changed = source_ref_id != source_commit.raw_id
             merge_check.source_commit.ref_spec = pull_request.source_ref_parts
             merge_check.source_commit.current_raw_id = source_commit.raw_id
             merge_check.source_commit.previous_raw_id = source_ref_id
 
             merge_check.target_commit.changed = target_ref_id != target_commit.raw_id
             merge_check.target_commit.ref_spec = pull_request.target_ref_parts
             merge_check.target_commit.current_raw_id = target_commit.raw_id
             merge_check.target_commit.previous_raw_id = target_ref_id
         except (SourceRefMissing, TargetRefMissing):
             pass
 
         if not merge_status:
             log.debug("MergeCheck: cannot merge, pull request merge not possible.")
             merge_check.push_error('warning', msg, cls.MERGE_CHECK, None)
 
             if fail_early:
                 return merge_check
 
         log.debug('MergeCheck: is failed: %s', merge_check.failed)
         return merge_check
 
     @classmethod
     def get_merge_conditions(cls, pull_request, translator):
         _ = translator
         merge_details = {}
 
         model = PullRequestModel()
         use_rebase = model._use_rebase_for_merging(pull_request)
 
         if use_rebase:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: rebase')
             )
         else:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: explicit merge commit')
             )
 
         close_branch = model._close_branch_before_merging(pull_request)
         if close_branch:
             repo_type = pull_request.target_repo.repo_type
             close_msg = ''
             if repo_type == 'hg':
                 close_msg = _('Source branch will be closed before the merge.')
             elif repo_type == 'git':
                 close_msg = _('Source branch will be deleted after the merge.')
 
             merge_details['close_branch'] = dict(
                 details={},
                 message=close_msg
             )
 
         return merge_details
 
 
 @dataclasses.dataclass
 class ChangeTuple:
     added: list
     common: list
     removed: list
     total: list
 
 
 @dataclasses.dataclass
 class FileChangeTuple:
     added: list
     modified: list
     removed: list

rhodecode/tests/lib/middleware/test_simplevcs.py

@@ -1,487 +1,451 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.lib.str_utils import base64_to_str
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.tests.utils import CustomTestApp
 
 from rhodecode.lib.caching_query import FromCache
-from rhodecode.lib.hooks_daemon import DummyHooksCallbackDaemon
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.model.db import User, _hash_key
 from rhodecode.model.meta import Session, cache as db_cache
 from rhodecode.tests import (
     HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
 from rhodecode.tests.lib.middleware import mock_scm_app
 
 
 class StubVCSController(simplevcs.SimpleVCS):
 
     SCM = 'hg'
     stub_response_body = tuple()
 
     def __init__(self, *args, **kwargs):
         super(StubVCSController, self).__init__(*args, **kwargs)
         self._action = 'pull'
         self._is_shadow_repo_dir = True
         self._name = HG_REPO
         self.set_repo_names(None)
 
     @property
     def is_shadow_repo_dir(self):
         return self._is_shadow_repo_dir
 
     def _get_repository_name(self, environ):
         return self._name
 
     def _get_action(self, environ):
         return self._action
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         def fake_app(environ, start_response):
             headers = [
                 ('Http-Accept', 'application/mercurial')
             ]
             start_response('200 OK', headers)
             return self.stub_response_body
         return fake_app
 
     def _create_config(self, extras, repo_name, scheme='http'):
         return None
 
 
 @pytest.fixture()
 def vcscontroller(baseapp, config_stub, request_stub):
     from rhodecode.config.middleware import ce_auth_resources
 
     config_stub.testing_securitypolicy()
     config_stub.include('rhodecode.authentication')
 
     for resource in ce_auth_resources:
         config_stub.include(resource)
 
     controller = StubVCSController(
         baseapp.config.get_settings(), request_stub.registry)
     app = HttpsFixup(controller, baseapp.config.get_settings())
     app = CustomTestApp(app)
 
     _remove_default_user_from_query_cache()
 
     # Sanity checks that things are set up correctly
     app.get('/' + HG_REPO, status=200)
 
     app.controller = controller
     return app
 
 
 def _remove_default_user_from_query_cache():
     user = User.get_default_user(cache=True)
     query = Session().query(User).filter(User.username == user.username)
     query = query.options(
         FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}"))
 
     db_cache.invalidate(
         query, {},
         FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}"))
 
     Session().expire(user)
 
 
 def test_handles_exceptions_during_permissions_checks(
         vcscontroller, disable_anonymous_user, enable_auth_plugins, test_user_factory):
 
     test_password = 'qweqwe'
     test_user = test_user_factory(password=test_password, extern_type='headers', extern_name='headers')
     test_username = test_user.username
 
     enable_auth_plugins.enable([
         'egg:rhodecode-enterprise-ce#headers',
         'egg:rhodecode-enterprise-ce#token',
         'egg:rhodecode-enterprise-ce#rhodecode'],
         override={
             'egg:rhodecode-enterprise-ce#headers': {'auth_headers_header': 'REMOTE_USER'}
         })
 
     user_and_pass = f'{test_username}:{test_password}'
     auth_password = base64_to_str(user_and_pass)
 
     extra_environ = {
         'AUTH_TYPE': 'Basic',
         'HTTP_AUTHORIZATION': f'Basic {auth_password}',
         'REMOTE_USER': test_username,
     }
 
     # Verify that things are hooked up correctly, we pass user with headers bound auth, and headers filled in
     vcscontroller.get('/', status=200, extra_environ=extra_environ)
 
     # Simulate trouble during permission checks
     with mock.patch('rhodecode.model.db.User.get_by_username',
                     side_effect=Exception('permission_error_test')) as get_user:
         # Verify that a correct 500 is returned and check that the expected
         # code path was hit.
         vcscontroller.get('/', status=500, extra_environ=extra_environ)
         assert get_user.called
 
 
 class StubFailVCSController(simplevcs.SimpleVCS):
     def _handle_request(self, environ, start_response):
         raise Exception("BOOM")
 
 
 @pytest.fixture(scope='module')
 def fail_controller(baseapp):
     controller = StubFailVCSController(
         baseapp.config.get_settings(), baseapp.config)
     controller = HttpsFixup(controller, baseapp.config.get_settings())
     controller = CustomTestApp(controller)
     return controller
 
 
 def test_handles_exceptions_as_internal_server_error(fail_controller):
     fail_controller.get('/', status=500)
 
 
 def test_provides_traceback_for_appenlight(fail_controller):
     response = fail_controller.get(
         '/', status=500, extra_environ={'appenlight.client': 'fake'})
     assert 'appenlight.__traceback' in response.request.environ
 
 
 def test_provides_utils_scm_app_as_scm_app_by_default(baseapp, request_stub):
     controller = StubVCSController(baseapp.config.get_settings(), request_stub.registry)
     assert controller.scm_app is scm_app_http
 
 
 def test_allows_to_override_scm_app_via_config(baseapp, request_stub):
     config = baseapp.config.get_settings().copy()
     config['vcs.scm_app_implementation'] = (
         'rhodecode.tests.lib.middleware.mock_scm_app')
     controller = StubVCSController(config, request_stub.registry)
     assert controller.scm_app is mock_scm_app
 
 
 @pytest.mark.parametrize('query_string, expected', [
     ('cmd=stub_command', True),
     ('cmd=listkeys', False),
 ])
 def test_should_check_locking(query_string, expected):
     result = simplevcs._should_check_locking(query_string)
     assert result == expected
 
 
 class TestShadowRepoRegularExpression(object):
     pr_segment = 'pull-request'
     shadow_segment = 'repository'
 
     @pytest.mark.parametrize('url, expected', [
         # repo with/without groups
         ('My-Repo/{pr_segment}/1/{shadow_segment}', True),
         ('Group/My-Repo/{pr_segment}/2/{shadow_segment}', True),
         ('Group/Sub-Group/My-Repo/{pr_segment}/3/{shadow_segment}', True),
         ('Group/Sub-Group1/Sub-Group2/My-Repo/{pr_segment}/3/{shadow_segment}', True),
 
         # pull request ID
         ('MyRepo/{pr_segment}/1/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/1234567890/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/-1/{shadow_segment}', False),
         ('MyRepo/{pr_segment}/invalid/{shadow_segment}', False),
 
         # unicode
         (u'Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
         (u'Sp€çîál-Gröüp/Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
 
         # trailing/leading slash
         ('/My-Repo/{pr_segment}/1/{shadow_segment}', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}/', False),
         ('/My-Repo/{pr_segment}/1/{shadow_segment}/', False),
 
         # misc
         ('My-Repo/{pr_segment}/1/{shadow_segment}/extra', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}extra', False),
     ])
     def test_shadow_repo_regular_expression(self, url, expected):
         from rhodecode.lib.middleware.simplevcs import SimpleVCS
         url = url.format(
             pr_segment=self.pr_segment,
             shadow_segment=self.shadow_segment)
         match_obj = SimpleVCS.shadow_repo_re.match(url)
         assert (match_obj is not None) == expected
 
 
 @pytest.mark.backends('git', 'hg')
 class TestShadowRepoExposure(object):
 
     def test_pull_on_shadow_repo_propagates_to_wsgi_app(
             self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = True
         controller.stub_response_body = (b'dummy body value',)
         controller._get_default_cache_ttl = mock.Mock(
             return_value=(False, 0))
 
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert response_body == b''.join(controller.stub_response_body)
 
     def test_pull_on_shadow_repo_that_is_missing(self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = False
         controller.stub_response_body = (b'dummy body value',)
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert b'404 Not Found' in response_body
 
     def test_push_on_shadow_repo_raises(self, baseapp, request_stub):
         """
         Check that a push action to a shadow repo is aborted.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'push'
         controller.stub_response_body = (b'dummy body value',)
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         assert response_body != controller.stub_response_body
         # Assert that a 406 error is returned.
         assert b'406 Not Acceptable' in response_body
 
     def test_set_repo_names_no_shadow(self, baseapp, request_stub):
         """
         Check that the set_repo_names method sets all names to the one returned
         by the _get_repository_name method on a request to a non shadow repo.
         """
         environ_stub = {}
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = 'RepoGroup/MyRepo'
         controller.set_repo_names(environ_stub)
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name ==
                 controller._get_repository_name(environ_stub))
 
     def test_set_repo_names_with_shadow(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Check that the set_repo_names method sets correct names on a request
         to a shadow repo.
         """
         from rhodecode.model.pull_request import PullRequestModel
 
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=pull_request.pull_request_id,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         # Get file system path to shadow repo for assertions.
         workspace_id = PullRequestModel()._workspace_id(pull_request)
         vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
         assert controller.vcs_repo_name == vcs_repo_name
         assert controller.url_repo_name == shadow_url
         assert controller.acl_repo_name == pull_request.target_repo.repo_name
         assert controller.is_shadow_repo
 
     def test_set_repo_names_with_shadow_but_missing_pr(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Checks that the set_repo_names method enforces matching target repos
         and pull request IDs.
         """
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=999999999,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name)
 
 
 @pytest.mark.usefixtures('baseapp')
 class TestGenerateVcsResponse(object):
 
     def test_ensures_that_start_response_is_called_early_enough(self):
         self.call_controller_with_response_body(iter(['a', 'b']))
         assert self.start_response.called
 
     def test_invalidates_cache_after_body_is_consumed(self):
         result = self.call_controller_with_response_body(iter(['a', 'b']))
         assert not self.was_cache_invalidated()
         # Consume the result
         list(result)
         assert self.was_cache_invalidated()
 
     def test_raises_unknown_exceptions(self):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='unknown'))
         with pytest.raises(Exception):
             list(result)
 
-    def test_prepare_callback_daemon_is_called(self):
-        def side_effect(extras, environ, action, txn_id=None):
-            return DummyHooksCallbackDaemon(), extras
-
-        prepare_patcher = mock.patch.object(
-            StubVCSController, '_prepare_callback_daemon')
-        with prepare_patcher as prepare_mock:
-            prepare_mock.side_effect = side_effect
-            self.call_controller_with_response_body(iter(['a', 'b']))
-        assert prepare_mock.called
-        assert prepare_mock.call_count == 1
-
     def call_controller_with_response_body(self, response_body):
         settings = {
             'base_path': 'fake_base_path',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.direct_calls': False,
         }
         registry = AttributeDict()
         controller = StubVCSController(settings, registry)
         controller._invalidate_cache = mock.Mock()
         controller.stub_response_body = response_body
         self.start_response = mock.Mock()
         result = controller._generate_vcs_response(
             environ={}, start_response=self.start_response,
             repo_path='fake_repo_path',
             extras={}, action='push')
         self.controller = controller
         return result
 
     def raise_result_iter(self, vcs_kind='repo_locked'):
         """
         Simulates an exception due to a vcs raised exception if kind vcs_kind
         """
         raise self.vcs_exception(vcs_kind=vcs_kind)
         yield "never_reached"
 
     def vcs_exception(self, vcs_kind='repo_locked'):
         locked_exception = Exception('TEST_MESSAGE')
         locked_exception._vcs_kind = vcs_kind
         return locked_exception
 
     def was_cache_invalidated(self):
         return self.controller._invalidate_cache.called
 
 
 class TestInitializeGenerator(object):
 
     def test_drains_first_element(self):
         gen = self.factory(['__init__', 1, 2])
         result = list(gen)
         assert result == [1, 2]
 
     @pytest.mark.parametrize('values', [
         [],
         [1, 2],
     ])
     def test_raises_value_error(self, values):
         with pytest.raises(ValueError):
             self.factory(values)
 
     @simplevcs.initialize_generator
     def factory(self, iterable):
         for elem in iterable:
             yield elem
-
-
-class TestPrepareHooksDaemon(object):
-    def test_calls_imported_prepare_callback_daemon(self, app_settings, request_stub):
-        expected_extras = {'extra1': 'value1'}
-        daemon = DummyHooksCallbackDaemon()
-
-        controller = StubVCSController(app_settings, request_stub.registry)
-        prepare_patcher = mock.patch.object(
-            simplevcs, 'prepare_callback_daemon',
-            return_value=(daemon, expected_extras))
-        with prepare_patcher as prepare_mock:
-            callback_daemon, extras = controller._prepare_callback_daemon(
-                expected_extras.copy(), {}, 'push')
-        prepare_mock.assert_called_once_with(
-            expected_extras,
-            protocol=app_settings['vcs.hooks.protocol'],
-            host=app_settings['vcs.hooks.host'],
-            txn_id=None,
-            use_direct_calls=app_settings['vcs.hooks.direct_calls'])
-
-        assert callback_daemon == daemon
-        assert extras == extras

rhodecode/tests/lib/test_hooks_daemon.py

@@ -1,376 +1,355 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import io
 
 import mock
 import msgpack
 import pytest
+import tempfile
 
 from rhodecode.lib import hooks_daemon
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.tests.utils import assert_message_in_log
 from rhodecode.lib.ext_json import json
 
 test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
 
 
-class TestDummyHooksCallbackDaemon(object):
-    def test_hooks_module_path_set_properly(self):
-        daemon = hooks_daemon.DummyHooksCallbackDaemon()
-        assert daemon.hooks_module == 'rhodecode.lib.hooks_daemon'
-
-    def test_logs_entering_the_hook(self):
-        daemon = hooks_daemon.DummyHooksCallbackDaemon()
-        with mock.patch.object(hooks_daemon.log, 'debug') as log_mock:
-            with daemon as return_value:
-                log_mock.assert_called_once_with(
-                    'Running `%s` callback daemon', 'DummyHooksCallbackDaemon')
-        assert return_value == daemon
-
-    def test_logs_exiting_the_hook(self):
-        daemon = hooks_daemon.DummyHooksCallbackDaemon()
-        with mock.patch.object(hooks_daemon.log, 'debug') as log_mock:
-            with daemon:
-                pass
-        log_mock.assert_called_with(
-            'Exiting `%s` callback daemon', 'DummyHooksCallbackDaemon')
-
-
 class TestHooks(object):
     def test_hooks_can_be_used_as_a_context_processor(self):
         hooks = hooks_daemon.Hooks()
         with hooks as return_value:
             pass
         assert hooks == return_value
 
 
 class TestHooksHttpHandler(object):
     def test_read_request_parses_method_name_and_arguments(self):
         data = {
             'method': 'test',
             'extras': {
                 'param1': 1,
                 'param2': 'a'
             }
         }
         request = self._generate_post_request(data)
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, data['method'], create=True, return_value=1)
 
         with hooks_patcher as hooks_mock:
             handler = hooks_daemon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             MockServer(handler, request)
 
         hooks_mock.assert_called_once_with(data['extras'])
 
     def test_hooks_serialized_result_is_returned(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
         hook_result = {
             'first': 'one',
             'second': 2
         }
         extras = {}
 
         # patching our _read to return test method and proto used
         read_patcher = mock.patch.object(
             hooks_daemon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, extras))
 
         # patch Hooks instance to return hook_result data on 'test' call
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, rpc_method, create=True,
             return_value=hook_result)
 
         with read_patcher, hooks_patcher:
             handler = hooks_daemon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
 
         expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result)
 
         server.request.output_stream.seek(0)
         assert server.request.output_stream.readlines()[-1] == expected_result
 
     def test_exception_is_returned_in_response(self):
         request = self._generate_post_request({})
         rpc_method = 'test'
 
         read_patcher = mock.patch.object(
             hooks_daemon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, {}))
 
         hooks_patcher = mock.patch.object(
             hooks_daemon.Hooks, rpc_method, create=True,
             side_effect=Exception('Test exception'))
 
         with read_patcher, hooks_patcher:
             handler = hooks_daemon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
 
         server.request.output_stream.seek(0)
         data = server.request.output_stream.readlines()
         msgpack_data = b''.join(data[5:])
         org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data)
         expected_result = {
             'exception': 'Exception',
             'exception_traceback': org_exc['exception_traceback'],
             'exception_args': ['Test exception']
         }
         assert org_exc == expected_result
 
     def test_log_message_writes_to_debug_log(self, caplog):
         ip_port = ('0.0.0.0', 8888)
         handler = hooks_daemon.HooksHttpHandler(
             MockRequest('POST /'), ip_port, mock.Mock())
         fake_date = '1/Nov/2015 00:00:00'
         date_patcher = mock.patch.object(
             handler, 'log_date_time_string', return_value=fake_date)
 
         with date_patcher, caplog.at_level(logging.DEBUG):
             handler.log_message('Some message %d, %s', 123, 'string')
 
         expected_message = f"HOOKS: client={ip_port} - - [{fake_date}] Some message 123, string"
 
         assert_message_in_log(
             caplog.records, expected_message,
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def _generate_post_request(self, data, proto=test_proto):
         if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
             payload = msgpack.packb(data)
         else:
             payload = json.dumps(data)
 
         return b'POST / HTTP/1.0\nContent-Length: %d\n\n%b' % (
             len(payload), payload)
 
 
 class ThreadedHookCallbackDaemon(object):
     def test_constructor_calls_prepare(self):
         prepare_daemon_patcher = mock.patch.object(
             hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
         with prepare_daemon_patcher as prepare_daemon_mock:
             hooks_daemon.ThreadedHookCallbackDaemon()
         prepare_daemon_mock.assert_called_once_with()
 
     def test_run_is_called_on_context_start(self):
         patchers = mock.patch.multiple(
             hooks_daemon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
 
         with patchers as mocks:
             daemon = hooks_daemon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 pass
             mocks['_run'].assert_called_once_with()
         assert daemon_context == daemon
 
     def test_stop_is_called_on_context_exit(self):
         patchers = mock.patch.multiple(
             hooks_daemon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
 
         with patchers as mocks:
             daemon = hooks_daemon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 assert mocks['_stop'].call_count == 0
 
             mocks['_stop'].assert_called_once_with()
         assert daemon_context == daemon
 
 
 class TestHttpHooksCallbackDaemon(object):
     def test_hooks_callback_generates_new_port(self, caplog):
         with caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon.server_address == ('127.0.0.1', 8881)
 
         with caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
         assert daemon._daemon.server_address[1] in range(0, 66000)
         assert daemon._daemon.server_address[0] != '127.0.0.1'
 
     def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon == tcp_server
 
         _, port = tcp_server.server_address
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
               f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
         assert_message_in_log(
             caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_prepare_inits_hooks_uri_and_logs_it(
             self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
             daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
 
         _, port = tcp_server.server_address
         expected_uri = '{}:{}'.format('127.0.0.1', port)
         assert daemon.hooks_uri == expected_uri
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
               f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
         assert_message_in_log(
             caplog.records, msg,
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_run_creates_a_thread(self, tcp_server):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread) as thread_mock:
             daemon._run()
 
         thread_mock.assert_called_once_with(
             target=tcp_server.serve_forever,
             kwargs={'poll_interval': daemon.POLL_INTERVAL})
         assert thread.daemon is True
         thread.start.assert_called_once_with()
 
     def test_run_logs(self, tcp_server, caplog):
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
             daemon._run()
 
         assert_message_in_log(
             caplog.records,
             'Running thread-based loop of callback daemon in background',
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
             daemon = hooks_daemon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
             with daemon:
                 assert daemon._daemon == tcp_server
                 assert daemon._callback_thread == thread
 
         assert daemon._daemon is None
         assert daemon._callback_thread is None
         tcp_server.shutdown.assert_called_with()
         thread.join.assert_called_once_with()
 
         assert_message_in_log(
             caplog.records, 'Waiting for background thread to finish.',
             levelno=logging.DEBUG, module='hooks_daemon')
 
     def _tcp_patcher(self, tcp_server):
         return mock.patch.object(
             hooks_daemon, 'TCPServer', return_value=tcp_server)
 
     def _thread_patcher(self, thread):
         return mock.patch.object(
             hooks_daemon.threading, 'Thread', return_value=thread)
 
 
 class TestPrepareHooksDaemon(object):
-    @pytest.mark.parametrize('protocol', ('http',))
-    def test_returns_dummy_hooks_callback_daemon_when_using_direct_calls(
+    @pytest.mark.parametrize('protocol', ('celery',))
+    def test_returns_celery_hooks_callback_daemon_when_celery_protocol_specified(
             self, protocol):
-        expected_extras = {'extra1': 'value1'}
-        callback, extras = hooks_daemon.prepare_callback_daemon(
-            expected_extras.copy(), protocol=protocol,
-            host='127.0.0.1', use_direct_calls=True)
-        assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
-        expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
-        expected_extras['time'] = extras['time']
-        assert 'extra1' in extras
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            temp_file.write("[app:main]\ncelery.broker_url = redis://redis/0\n"
+                            "celery.result_backend = redis://redis/0")
+            temp_file.flush()
+            expected_extras = {'config': temp_file.name}
+            callback, extras = hooks_daemon.prepare_callback_daemon(
+                expected_extras, protocol=protocol, host='')
+            assert isinstance(callback, hooks_daemon.CeleryHooksCallbackDaemon)
 
     @pytest.mark.parametrize('protocol, expected_class', (
         ('http', hooks_daemon.HttpHooksCallbackDaemon),
     ))
     def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
             self, protocol, expected_class):
         expected_extras = {
             'extra1': 'value1',
             'txn_id': 'txnid2',
-            'hooks_protocol': protocol.lower()
+            'hooks_protocol': protocol.lower(),
+            'task_backend': '',
+            'task_queue': ''
         }
         callback, extras = hooks_daemon.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
-            use_direct_calls=False,
             txn_id='txnid2')
         assert isinstance(callback, expected_class)
         extras.pop('hooks_uri')
         expected_extras['time'] = extras['time']
         assert extras == expected_extras
 
     @pytest.mark.parametrize('protocol', (
         'invalid',
         'Http',
         'HTTP',
     ))
     def test_raises_on_invalid_protocol(self, protocol):
         expected_extras = {
             'extra1': 'value1',
             'hooks_protocol': protocol.lower()
         }
         with pytest.raises(Exception):
             callback, extras = hooks_daemon.prepare_callback_daemon(
                 expected_extras.copy(),
-                protocol=protocol, host='127.0.0.1',
-                use_direct_calls=False)
+                protocol=protocol, host='127.0.0.1')
 
 
 class MockRequest(object):
 
     def __init__(self, request):
         self.request = request
         self.input_stream = io.BytesIO(safe_bytes(self.request))
         self.output_stream = io.BytesIO()  # make it un-closable for testing invesitagion
         self.output_stream.close = lambda: None
 
     def makefile(self, mode, *args, **kwargs):
         return self.output_stream if mode == 'wb' else self.input_stream
 
 
 class MockServer(object):
 
     def __init__(self, handler_cls, request):
         ip_port = ('0.0.0.0', 8888)
         self.request = MockRequest(request)
         self.server_address = ip_port
         self.handler = handler_cls(self.request, ip_port, self)
 
 
 @pytest.fixture()
 def tcp_server():
     server = mock.Mock()
     server.server_address = ('127.0.0.1', 8881)
     server.wbufsize = 1024
     return server