fix(server-info): removed dulwich from update-server-info and added force flag support

rhodecode/lib/utils.py

@@ -1,808 +1,808 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
 import warnings
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
 
 from mako import exceptions
 
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     replacement_char = '-'
 
     slug = strip_tags(value)
     slug = convert_accented_entities(slug)
     slug = convert_misc_entities(slug)
 
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub(r'[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
 
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo_name'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     yield from _get_repos(rec_path)
 
     return _get_repos(path)
 
 
 def get_dirpaths(p: str) -> list:
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 "Ignoring path %s since it cannot be decoded into str.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if a given path is a repository group, False otherwise
 
     :param repo_group_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check; we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise OSError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
     for setting in ui_settings:
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in list(app_settings.items()):
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
     return rhodecode.CONFIG['default_base_path']
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in list(initial_repo_list.items()):
         group = map_groups(name)
         str_name = safe_str(name)
         db_repo = repo_model.get_by_repo_name(str_name)
 
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
-                git_repo._update_server_info()
+                git_repo._update_server_info(force=True)
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks(force=force_hooks_rebuild)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in list(initial_repo_list.keys()):
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    list(initial_repo_list.keys()) if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         def nested_sort(gr):
             return len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     try:
         import rc_testdata
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.lib.utils2 import get_encryption_key
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
 
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return sha256_safe(':'.join(uuid_list))
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)

rhodecode/lib/vcs/backends/git/repository.py

@@ -1,1053 +1,1053 b''
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT repository module
 """
 
 import logging
 import os
 import re
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from collections import OrderedDict
 from rhodecode.lib.datelib import (
     utcdate_fromtimestamp, makedate, date_astimestamp)
 from rhodecode.lib.hash_utils import safe_str
 from rhodecode.lib.utils2 import CachedProperty
 from rhodecode.lib.vcs import connection, path as vcspath
 from rhodecode.lib.vcs.backends.base import (
     BaseRepository, CollectionGenerator, Config, MergeResponse,
     MergeFailureReason, Reference)
 from rhodecode.lib.vcs.backends.git.commit import GitCommit
 from rhodecode.lib.vcs.backends.git.diff import GitDiff
 from rhodecode.lib.vcs.backends.git.inmemory import GitInMemoryCommit
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError,
     RepositoryError, TagAlreadyExistError, TagDoesNotExistError, VCSError, UnresolvedFilesInRepo)
 
 
 SHA_PATTERN = re.compile(r'^([0-9a-fA-F]{12}|[0-9a-fA-F]{40})$')
 
 log = logging.getLogger(__name__)
 
 
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = os.environ.get('GIT_DEFAULT_BRANCH_NAME') or 'master'
     DEFAULT_REF = f'branch:{DEFAULT_BRANCH_NAME}'
 
     contact = BaseRepository.DEFAULT_CONTACT
 
     def __init__(self, repo_path, config=None, create=False, src_url=None,
                  do_workspace_checkout=False, with_wire=None, bare=False):
 
         self.path = safe_str(os.path.abspath(repo_path))
         self.config = config if config else self.get_default_config()
         self.with_wire = with_wire or {"cache": False}  # default should not use cache
 
         self._init_repo(create, src_url, do_workspace_checkout, bare)
 
         # caches
         self._commit_ids = {}
 
     @LazyProperty
     def _remote(self):
         repo_id = self.path
         return connection.Git(self.path, repo_id, self.config, with_wire=self.with_wire)
 
     @LazyProperty
     def bare(self):
         return self._remote.bare()
 
     @LazyProperty
     def head(self):
         return self._remote.head()
 
     @CachedProperty
     def commit_ids(self):
         """
         Returns list of commit ids, in ascending order.  Being lazy
         attribute allows external tools to inject commit ids from cache.
         """
         commit_ids = self._get_all_commit_ids()
         self._rebuild_cache(commit_ids)
         return commit_ids
 
     def _rebuild_cache(self, commit_ids):
         self._commit_ids = {commit_id: index
                                 for index, commit_id in enumerate(commit_ids)}
 
     def run_git_command(self, cmd, **opts):
         """
         Runs given ``cmd`` as git command and returns tuple
         (stdout, stderr).
 
         :param cmd: git command to be executed
         :param opts: env options to pass into Subprocess command
         """
         if not isinstance(cmd, list):
             raise ValueError(f'cmd must be a list, got {type(cmd)} instead')
 
         skip_stderr_log = opts.pop('skip_stderr_log', False)
         out, err = self._remote.run_git_command(cmd, **opts)
         if err and not skip_stderr_log:
             log.debug('Stderr output of git command "%s":\n%s', cmd, err)
         return out, err
 
     @staticmethod
     def check_url(url, config):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         if '+' in url.split('://', 1)[0]:
             url = url.split('+', 1)[1]
 
         # Request the _remote to verify the url
         return connection.Git.check_url(url, config.serialize())
 
     @staticmethod
     def is_valid_repository(path):
         if os.path.isdir(os.path.join(path, '.git')):
             return True
         # check case of bare repository
         try:
             GitRepository(path)
             return True
         except VCSError:
             pass
         return False
 
     def _init_repo(self, create, src_url=None, do_workspace_checkout=False,
                    bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError(
                 f"Cannot create repository at {self.path}, location already exist")
 
         if bare and do_workspace_checkout:
             raise RepositoryError("Cannot update a bare repository")
         try:
 
             if src_url:
                 # check URL before any actions
                 GitRepository.check_url(src_url, self.config)
 
             if create:
                 if bare:
                     self._remote.init_bare()
                 else:
                     self._remote.init()
 
                 if src_url and bare:
                     # bare repository only allows a fetch and checkout is not allowed
                     self.fetch(src_url, commit_ids=None)
                 elif src_url:
                     self.pull(src_url, commit_ids=None,
                               update_after=do_workspace_checkout)
 
             else:
                 if not self._remote.assert_correct_path():
                     raise RepositoryError(
                         f'Path "{self.path}" does not contain a Git repository')
 
         # TODO: johbo: check if we have to translate the OSError here
         except OSError as err:
             raise RepositoryError(err)
 
     def _get_all_commit_ids(self):
         return self._remote.get_all_commit_ids()
 
     def _get_commit_ids(self, filters=None):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
 
         head = self._remote.head(show_exc=False)
 
         if not head:
             return []
 
         rev_filter = ['--branches', '--tags']
         extra_filter = []
 
         if filters:
             if filters.get('since'):
                 extra_filter.append('--since=%s' % (filters['since']))
             if filters.get('until'):
                 extra_filter.append('--until=%s' % (filters['until']))
             if filters.get('branch_name'):
                 rev_filter = []
                 extra_filter.append(filters['branch_name'])
         rev_filter.extend(extra_filter)
 
             # if filters.get('start') or filters.get('end'):
             #     # skip is offset, max-count is limit
             #     if filters.get('start'):
             #         extra_filter += ' --skip=%s' % filters['start']
             #     if filters.get('end'):
             #         extra_filter += ' --max-count=%s' % (filters['end'] - (filters['start'] or 0))
 
         cmd = ['rev-list', '--reverse', '--date-order'] + rev_filter
         try:
             output, __ = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return output.splitlines()
 
     def _lookup_commit(self, commit_id_or_idx, translate_tag=True, maybe_unreachable=False, reference_obj=None):
 
         def is_null(value):
             return len(value) == commit_id_or_idx.count('0')
 
         if commit_id_or_idx in (None, '', 'tip', 'HEAD', 'head', -1):
             return self.commit_ids[-1]
 
         commit_missing_err = "Commit {} does not exist for `{}`".format(
             *map(safe_str, [commit_id_or_idx, self.name]))
 
         is_bstr = isinstance(commit_id_or_idx, str)
         is_branch = reference_obj and reference_obj.branch
 
         lookup_ok = False
         if is_bstr:
             # Need to call remote to translate id for tagging scenarios,
             # or branch that are numeric
             try:
                 remote_data = self._remote.get_object(commit_id_or_idx,
                                                       maybe_unreachable=maybe_unreachable)
                 commit_id_or_idx = remote_data["commit_id"]
                 lookup_ok = True
             except (CommitDoesNotExistError,):
                 lookup_ok = False
 
         if lookup_ok is False:
             is_numeric_idx = \
                 (is_bstr and commit_id_or_idx.isdigit() and len(commit_id_or_idx) < 12) \
                 or isinstance(commit_id_or_idx, int)
             if not is_branch and (is_numeric_idx or is_null(commit_id_or_idx)):
                 try:
                     commit_id_or_idx = self.commit_ids[int(commit_id_or_idx)]
                     lookup_ok = True
                 except Exception:
                     raise CommitDoesNotExistError(commit_missing_err)
 
         # we failed regular lookup, and by integer number lookup
         if lookup_ok is False:
             raise CommitDoesNotExistError(commit_missing_err)
 
         # Ensure we return full id
         if not SHA_PATTERN.match(str(commit_id_or_idx)):
             raise CommitDoesNotExistError(
                 "Given commit id %s not recognized" % commit_id_or_idx)
         return commit_id_or_idx
 
     def get_hook_location(self):
         """
         returns absolute path to location where hooks are stored
         """
         loc = os.path.join(self.path, 'hooks')
         if not self.bare:
             loc = os.path.join(self.path, '.git', 'hooks')
         return loc
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as
         `datetime.datetime` object.
         """
         try:
             return self.get_commit().date
         except RepositoryError:
             tzoffset = makedate()[1]
             return utcdate_fromtimestamp(self._get_fs_mtime(), tzoffset)
 
     def _get_fs_mtime(self):
         idx_loc = '' if self.bare else '.git'
         # fallback to filesystem
         in_path = os.path.join(self.path, idx_loc, "index")
         he_path = os.path.join(self.path, idx_loc, "HEAD")
         if os.path.exists(in_path):
             return os.stat(in_path).st_mtime
         else:
             return os.stat(he_path).st_mtime
 
     @LazyProperty
     def description(self):
         description = self._remote.get_description()
         return safe_str(description or self.DEFAULT_DESCRIPTION)
 
     def _get_refs_entries(self, prefix='', reverse=False, strip_prefix=True):
         if self.is_empty():
             return OrderedDict()
 
         result = []
         for ref, sha in self._refs.items():
             if ref.startswith(prefix):
                 ref_name = ref
                 if strip_prefix:
                     ref_name = ref[len(prefix):]
                 result.append((safe_str(ref_name), sha))
 
         def get_name(entry):
             return entry[0]
 
         return OrderedDict(sorted(result, key=get_name, reverse=reverse))
 
     def _get_branches(self):
         return self._get_refs_entries(prefix='refs/heads/', strip_prefix=True)
 
     @CachedProperty
     def branches(self):
         return self._get_branches()
 
     @CachedProperty
     def branches_closed(self):
         return {}
 
     @CachedProperty
     def bookmarks(self):
         return {}
 
     @CachedProperty
     def branches_all(self):
         all_branches = {}
         all_branches.update(self.branches)
         all_branches.update(self.branches_closed)
         return all_branches
 
     @CachedProperty
     def tags(self):
         return self._get_tags()
 
     def _get_tags(self):
         return self._get_refs_entries(prefix='refs/tags/', strip_prefix=True, reverse=True)
 
     def tag(self, name, user, commit_id=None, message=None, date=None,
             **kwargs):
         # TODO: fix this method to apply annotated tags correct with message
         """
         Creates and returns a tag for the given ``commit_id``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param commit_id: commit id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         commit = self.get_commit(commit_id=commit_id)
         message = message or f"Added tag {name} for commit {commit.raw_id}"
 
         self._remote.set_refs('refs/tags/%s' % name, commit.raw_id)
 
         self._invalidate_prop_cache('tags')
         self._invalidate_prop_cache('_refs')
 
         return commit
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
 
         self._remote.tag_remove(name)
         self._invalidate_prop_cache('tags')
         self._invalidate_prop_cache('_refs')
 
     def _get_refs(self):
         return self._remote.get_refs()
 
     @CachedProperty
     def _refs(self):
         return self._get_refs()
 
     @property
     def _ref_tree(self):
         node = tree = {}
         for ref, sha in self._refs.items():
             path = ref.split('/')
             for bit in path[:-1]:
                 node = node.setdefault(bit, {})
             node[path[-1]] = sha
             node = tree
         return tree
 
     def get_remote_ref(self, ref_name):
         ref_key = f'refs/remotes/origin/{safe_str(ref_name)}'
         try:
             return self._refs[ref_key]
         except Exception:
             return
 
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None,
                    translate_tag=True, maybe_unreachable=False, reference_obj=None):
         """
         Returns `GitCommit` object representing commit from git repository
         at the given `commit_id` or head (most recent commit) if None given.
         """
 
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
 
         if commit_id is not None:
             self._validate_commit_id(commit_id)
             try:
                 # we have cached idx, use it without contacting the remote
                 idx = self._commit_ids[commit_id]
                 return GitCommit(self, commit_id, idx, pre_load=pre_load)
             except KeyError:
                 pass
 
         elif commit_idx is not None:
             self._validate_commit_idx(commit_idx)
             try:
                 _commit_id = self.commit_ids[commit_idx]
                 if commit_idx < 0:
                     commit_idx = self.commit_ids.index(_commit_id)
                 return GitCommit(self, _commit_id, commit_idx, pre_load=pre_load)
             except IndexError:
                 commit_id = commit_idx
         else:
             commit_id = "tip"
 
         if translate_tag:
             commit_id = self._lookup_commit(
                 commit_id, maybe_unreachable=maybe_unreachable,
                 reference_obj=reference_obj)
 
         try:
             idx = self._commit_ids[commit_id]
         except KeyError:
             idx = -1
 
         return GitCommit(self, commit_id, idx, pre_load=pre_load)
 
     def get_commits(
             self, start_id=None, end_id=None, start_date=None, end_date=None,
             branch_name=None, show_hidden=False, pre_load=None, translate_tags=True):
         """
         Returns generator of `GitCommit` objects from start to end (both
         are inclusive), in ascending date order.
 
         :param start_id: None, str(commit_id)
         :param end_id: None, str(commit_id)
         :param start_date: if specified, commits with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, commits with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, commits not reachable from given
           branch would be filtered out from returned set
         :param show_hidden: Show hidden commits such as obsolete or hidden from
             Mercurial evolve
         :raise BranchDoesNotExistError: If given `branch_name` does not
             exist.
         :raise CommitDoesNotExistError: If commits for given `start` or
           `end` could not be found.
 
         """
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
 
         self._validate_branch_name(branch_name)
 
         if start_id is not None:
             self._validate_commit_id(start_id)
         if end_id is not None:
             self._validate_commit_id(end_id)
 
         start_raw_id = self._lookup_commit(start_id)
         start_pos = self._commit_ids[start_raw_id] if start_id else None
         end_raw_id = self._lookup_commit(end_id)
         end_pos = max(0, self._commit_ids[end_raw_id]) if end_id else None
 
         if None not in [start_id, end_id] and start_pos > end_pos:
             raise RepositoryError(
                 "Start commit '%s' cannot be after end commit '%s'" %
                 (start_id, end_id))
 
         if end_pos is not None:
             end_pos += 1
 
         filter_ = []
         if branch_name:
             filter_.append({'branch_name': branch_name})
         if start_date and not end_date:
             filter_.append({'since': start_date})
         if end_date and not start_date:
             filter_.append({'until': end_date})
         if start_date and end_date:
             filter_.append({'since': start_date})
             filter_.append({'until': end_date})
 
         # if start_pos or end_pos:
         #     filter_.append({'start': start_pos})
         #     filter_.append({'end': end_pos})
 
         if filter_:
             revfilters = {
                 'branch_name': branch_name,
                 'since': start_date.strftime('%m/%d/%y %H:%M:%S') if start_date else None,
                 'until': end_date.strftime('%m/%d/%y %H:%M:%S') if end_date else None,
                 'start': start_pos,
                 'end': end_pos,
             }
             commit_ids = self._get_commit_ids(filters=revfilters)
 
         else:
             commit_ids = self.commit_ids
 
         if start_pos or end_pos:
             commit_ids = commit_ids[start_pos: end_pos]
 
         return CollectionGenerator(self, commit_ids, pre_load=pre_load,
                                    translate_tag=translate_tags)
 
     def get_diff(
             self, commit1, commit2, path='', ignore_whitespace=False,
             context=3, path1=None):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``commit2`` since ``commit1``.
 
         :param commit1: Entry point from which diff is shown. Can be
           ``self.EMPTY_COMMIT`` - in this case, patch showing all
           the changes since empty state of the repository until ``commit2``
         :param commit2: Until which commits changes should be shown.
         :param path:
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         :param path1:
         """
         self._validate_diff_commits(commit1, commit2)
         if path1 is not None and path1 != path:
             raise ValueError("Diff of two different paths not supported.")
 
         if path:
             file_filter = path
         else:
             file_filter = None
 
         diff = self._remote.diff(
             commit1.raw_id, commit2.raw_id, file_filter=file_filter,
             opt_ignorews=ignore_whitespace,
             context=context)
 
         return GitDiff(diff)
 
     def strip(self, commit_id, branch_name):
         commit = self.get_commit(commit_id=commit_id)
         if commit.merge:
             raise Exception('Cannot reset to merge commit')
 
         if not branch_name:
             raise ValueError(f'git strip requires a valid branch name, got {branch_name} instead')
 
         # parent is going to be the new head now
         commit = commit.parents[0]
         self._remote.update_refs(f'refs/heads/{branch_name}', commit.raw_id)
 
         # clear cached properties
         self._invalidate_prop_cache('commit_ids')
         self._invalidate_prop_cache('_refs')
         self._invalidate_prop_cache('branches')
 
         return len(self.commit_ids)
 
     def get_common_ancestor(self, commit_id1, commit_id2, repo2):
         log.debug('Calculating common ancestor between %sc1:%s and %sc2:%s',
                   self, commit_id1, repo2, commit_id2)
 
         if commit_id1 == commit_id2:
             return commit_id1
 
         if self != repo2:
             commits = self._remote.get_missing_revs(
                 commit_id1, commit_id2, repo2.path)
             if commits:
                 commit = repo2.get_commit(commits[-1])
                 if commit.parents:
                     ancestor_id = commit.parents[0].raw_id
                 else:
                     ancestor_id = None
             else:
                 # no commits from other repo, ancestor_id is the commit_id2
                 ancestor_id = commit_id2
         else:
             output, __ = self.run_git_command(
                 ['merge-base', commit_id1, commit_id2])
             ancestor_id = self.COMMIT_ID_PAT.findall(output)[0]
 
         log.debug('Found common ancestor with sha: %s', ancestor_id)
 
         return ancestor_id
 
     def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None):
         repo1 = self
         ancestor_id = None
 
         if commit_id1 == commit_id2:
             commits = []
         elif repo1 != repo2:
             missing_ids = self._remote.get_missing_revs(commit_id1, commit_id2,
                                                         repo2.path)
             commits = [
                 repo2.get_commit(commit_id=commit_id, pre_load=pre_load)
                 for commit_id in reversed(missing_ids)]
         else:
             output, __ = repo1.run_git_command(
                 ['log', '--reverse', '--pretty=format: %H', '-s',
                  f'{commit_id1}..{commit_id2}'])
             commits = [
                 repo1.get_commit(commit_id=commit_id, pre_load=pre_load)
                 for commit_id in self.COMMIT_ID_PAT.findall(output)]
 
         return commits
 
     @LazyProperty
     def in_memory_commit(self):
         """
         Returns ``GitInMemoryCommit`` object for this repository.
         """
         return GitInMemoryCommit(self)
 
     def pull(self, url, commit_ids=None, update_after=False):
         """
         Pull changes from external location. Pull is different in GIT
         that fetch since it's doing a checkout
 
         :param commit_ids: Optional. Can be set to a list of commit ids
            which shall be pulled from the other repository.
         """
         refs = None
         if commit_ids is not None:
             remote_refs = self._remote.get_remote_refs(url)
             refs = [ref for ref in remote_refs if remote_refs[ref] in commit_ids]
         self._remote.pull(url, refs=refs, update_after=update_after)
         self._remote.invalidate_vcs_cache()
 
     def fetch(self, url, commit_ids=None, **kwargs):
         """
         Fetch all git objects from external location.
         """
         self._remote.sync_fetch(url, refs=commit_ids, **kwargs)
         self._remote.invalidate_vcs_cache()
 
     def push(self, url, **kwargs):
         refs = None
         self._remote.sync_push(url, refs=refs, **kwargs)
 
     def set_refs(self, ref_name, commit_id):
         self._remote.set_refs(ref_name, commit_id)
         self._invalidate_prop_cache('_refs')
 
     def remove_ref(self, ref_name):
         self._remote.remove_ref(ref_name)
         self._invalidate_prop_cache('_refs')
 
     def run_gc(self, prune=True):
         cmd = ['gc', '--aggressive']
         if prune:
             cmd += ['--prune=now']
         _stdout, stderr = self.run_git_command(cmd, fail_on_stderr=False)
         return stderr
 
-    def _update_server_info(self):
+    def _update_server_info(self, force=False):
         """
         runs gits update-server-info command in this repo instance
         """
-        self._remote.update_server_info()
+        self._remote.update_server_info(force=force)
 
     def _current_branch(self):
         """
         Return the name of the current branch.
 
         It only works for non bare repositories (i.e. repositories with a
         working copy)
         """
         if self.bare:
             raise RepositoryError('Bare git repos do not have active branches')
 
         if self.is_empty():
             return None
 
         stdout, _ = self.run_git_command(['rev-parse', '--abbrev-ref', 'HEAD'])
         return stdout.strip()
 
     def _checkout(self, branch_name, create=False, force=False):
         """
         Checkout a branch in the working directory.
 
         It tries to create the branch if create is True, failing if the branch
         already exists.
 
         It only works for non bare repositories (i.e. repositories with a
         working copy)
         """
         if self.bare:
             raise RepositoryError('Cannot checkout branches in a bare git repo')
 
         cmd = ['checkout']
         if force:
             cmd.append('-f')
         if create:
             cmd.append('-b')
         cmd.append(branch_name)
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _create_branch(self, branch_name, commit_id):
         """
         creates a branch in a GIT repo
         """
         self._remote.create_branch(branch_name, commit_id)
 
     def _identify(self):
         """
         Return the current state of the working directory.
         """
         if self.bare:
             raise RepositoryError('Bare git repos do not have active branches')
 
         if self.is_empty():
             return None
 
         stdout, _ = self.run_git_command(['rev-parse', 'HEAD'])
         return stdout.strip()
 
     def _local_clone(self, clone_path, branch_name, source_branch=None):
         """
         Create a local clone of the current repo.
         """
         # N.B.(skreft): the --branch option is required as otherwise the shallow
         # clone will only fetch the active branch.
         cmd = ['clone', '--branch', branch_name,
                self.path, os.path.abspath(clone_path)]
 
         self.run_git_command(cmd, fail_on_stderr=False)
 
         # if we get the different source branch, make sure we also fetch it for
         # merge conditions
         if source_branch and source_branch != branch_name:
             # check if the ref exists.
             shadow_repo = GitRepository(os.path.abspath(clone_path))
             if shadow_repo.get_remote_ref(source_branch):
                 cmd = ['fetch', self.path, source_branch]
                 self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_fetch(self, repository_path, branch_name, use_origin=False):
         """
         Fetch a branch from a local repository.
         """
         repository_path = os.path.abspath(repository_path)
         if repository_path == self.path:
             raise ValueError('Cannot fetch from the same repository')
 
         if use_origin:
             branch_name = '+{branch}:refs/heads/{branch}'.format(
                 branch=branch_name)
 
         cmd = ['fetch', '--no-tags', '--update-head-ok',
                repository_path, branch_name]
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_reset(self, branch_name):
         branch_name = f'{branch_name}'
         cmd = ['reset', '--hard', branch_name, '--']
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _last_fetch_heads(self):
         """
         Return the last fetched heads that need merging.
 
         The algorithm is defined at
         https://github.com/git/git/blob/v2.1.3/git-pull.sh#L283
         """
         if not self.bare:
             fetch_heads_path = os.path.join(self.path, '.git', 'FETCH_HEAD')
         else:
             fetch_heads_path = os.path.join(self.path, 'FETCH_HEAD')
 
         heads = []
         with open(fetch_heads_path) as f:
             for line in f:
                 if '    not-for-merge   ' in line:
                     continue
                 line = re.sub('\t.*', '', line, flags=re.DOTALL)
                 heads.append(line)
 
         return heads
 
     def get_shadow_instance(self, shadow_repository_path, enable_hooks=False, cache=False):
         return GitRepository(shadow_repository_path, with_wire={"cache": cache})
 
     def _local_pull(self, repository_path, branch_name, ff_only=True):
         """
         Pull a branch from a local repository.
         """
         if self.bare:
             raise RepositoryError('Cannot pull into a bare git repository')
         # N.B.(skreft): The --ff-only option is to make sure this is a
         # fast-forward (i.e., we are only pulling new changes and there are no
         # conflicts with our current branch)
         # Additionally, that option needs to go before --no-tags, otherwise git
         # pull complains about it being an unknown flag.
         cmd = ['pull']
         if ff_only:
             cmd.append('--ff-only')
         cmd.extend(['--no-tags', repository_path, branch_name])
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_merge(self, merge_message, user_name, user_email, heads):
         """
         Merge the given head into the checked out branch.
 
         It will force a merge commit.
 
         Currently it raises an error if the repo is empty, as it is not possible
         to create a merge commit in an empty repo.
 
         :param merge_message: The message to use for the merge commit.
         :param heads: the heads to merge.
         """
         if self.bare:
             raise RepositoryError('Cannot merge into a bare git repository')
 
         if not heads:
             return
 
         if self.is_empty():
             # TODO(skreft): do something more robust in this case.
             raise RepositoryError('Do not know how to merge into empty repositories yet')
         unresolved = None
 
         # N.B.(skreft): the --no-ff option is used to enforce the creation of a
         # commit message. We also specify the user who is doing the merge.
         cmd = ['-c', f'user.name="{user_name}"',
                '-c', f'user.email={user_email}',
                'merge', '--no-ff', '-m', safe_str(merge_message)]
 
         merge_cmd = cmd + heads
 
         try:
             self.run_git_command(merge_cmd, fail_on_stderr=False)
         except RepositoryError:
             files = self.run_git_command(['diff', '--name-only', '--diff-filter', 'U'],
                                          fail_on_stderr=False)[0].splitlines()
             # NOTE(marcink): we add U notation for consistent with HG backend output
             unresolved = [f'U {f}' for f in files]
 
             # Cleanup any merge leftovers
             self._remote.invalidate_vcs_cache()
             self.run_git_command(['merge', '--abort'], fail_on_stderr=False)
 
             if unresolved:
                 raise UnresolvedFilesInRepo(unresolved)
             else:
                 raise
 
     def _local_push(
             self, source_branch, repository_path, target_branch,
             enable_hooks=False, rc_scm_data=None):
         """
         Push the source_branch to the given repository and target_branch.
 
         Currently it if the target_branch is not master and the target repo is
         empty, the push will work, but then GitRepository won't be able to find
         the pushed branch or the commits. As the HEAD will be corrupted (i.e.,
         pointing to master, which does not exist).
 
         It does not run the hooks in the target repo.
         """
         # TODO(skreft): deal with the case in which the target repo is empty,
         # and the target_branch is not master.
         target_repo = GitRepository(repository_path)
         if (not target_repo.bare and
                 target_repo._current_branch() == target_branch):
             # Git prevents pushing to the checked out branch, so simulate it by
             # pulling into the target repository.
             target_repo._local_pull(self.path, source_branch)
         else:
             cmd = ['push', os.path.abspath(repository_path),
                    f'{source_branch}:{target_branch}']
             gitenv = {}
             if rc_scm_data:
                 gitenv.update({'RC_SCM_DATA': rc_scm_data})
 
             if not enable_hooks:
                 gitenv['RC_SKIP_HOOKS'] = '1'
             self.run_git_command(cmd, fail_on_stderr=False, extra_env=gitenv)
 
     def _get_new_pr_branch(self, source_branch, target_branch):
         prefix = f'pr_{source_branch}-{target_branch}_'
         pr_branches = []
         for branch in self.branches:
             if branch.startswith(prefix):
                 pr_branches.append(int(branch[len(prefix):]))
 
         if not pr_branches:
             branch_id = 0
         else:
             branch_id = max(pr_branches) + 1
 
         return '%s%d' % (prefix, branch_id)
 
     def _maybe_prepare_merge_workspace(
             self, repo_id, workspace_id, target_ref, source_ref):
         shadow_repository_path = self._get_shadow_repository_path(
             self.path, repo_id, workspace_id)
         if not os.path.exists(shadow_repository_path):
             self._local_clone(
                 shadow_repository_path, target_ref.name, source_ref.name)
             log.debug('Prepared %s shadow repository in %s',
                       self.alias, shadow_repository_path)
 
         return shadow_repository_path
 
     def _merge_repo(self, repo_id, workspace_id, target_ref,
                     source_repo, source_ref, merge_message,
                     merger_name, merger_email, dry_run=False,
                     use_rebase=False, close_branch=False):
 
         log.debug('Executing merge_repo with %s strategy, dry_run mode:%s',
                   'rebase' if use_rebase else 'merge', dry_run)
 
         if target_ref.commit_id != self.branches[target_ref.name]:
             log.warning('Target ref %s commit mismatch %s vs %s', target_ref,
                         target_ref.commit_id, self.branches[target_ref.name])
             return MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD,
                 metadata={'target_ref': target_ref})
 
         shadow_repository_path = self._maybe_prepare_merge_workspace(
             repo_id, workspace_id, target_ref, source_ref)
         shadow_repo = self.get_shadow_instance(shadow_repository_path)
 
         # checkout source, if it's different. Otherwise we could not
         # fetch proper commits for merge testing
         if source_ref.name != target_ref.name:
             if shadow_repo.get_remote_ref(source_ref.name):
                 shadow_repo._checkout(source_ref.name, force=True)
 
         # checkout target, and fetch changes
         shadow_repo._checkout(target_ref.name, force=True)
 
         # fetch/reset pull the target, in case it is changed
         # this handles even force changes
         shadow_repo._local_fetch(self.path, target_ref.name, use_origin=True)
         shadow_repo._local_reset(target_ref.name)
 
         # Need to reload repo to invalidate the cache, or otherwise we cannot
         # retrieve the last target commit.
         shadow_repo = self.get_shadow_instance(shadow_repository_path)
         if target_ref.commit_id != shadow_repo.branches[target_ref.name]:
             log.warning('Shadow Target ref %s commit mismatch %s vs %s',
                         target_ref, target_ref.commit_id,
                         shadow_repo.branches[target_ref.name])
             return MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD,
                 metadata={'target_ref': target_ref})
 
         # calculate new branch
         pr_branch = shadow_repo._get_new_pr_branch(
             source_ref.name, target_ref.name)
         log.debug('using pull-request merge branch: `%s`', pr_branch)
         # checkout to temp branch, and fetch changes
         shadow_repo._checkout(pr_branch, create=True)
         try:
             shadow_repo._local_fetch(source_repo.path, source_ref.name)
         except RepositoryError:
             log.exception('Failure when doing local fetch on '
                           'shadow repo: %s', shadow_repo)
             return MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_SOURCE_REF,
                 metadata={'source_ref': source_ref})
 
         merge_ref = None
         merge_failure_reason = MergeFailureReason.NONE
         metadata = {}
         try:
             shadow_repo._local_merge(merge_message, merger_name, merger_email,
                                      [source_ref.commit_id])
             merge_possible = True
 
             # Need to invalidate the cache, or otherwise we
             # cannot retrieve the merge commit.
             shadow_repo = shadow_repo.get_shadow_instance(shadow_repository_path)
             merge_commit_id = shadow_repo.branches[pr_branch]
 
             # Set a reference pointing to the merge commit. This reference may
             # be used to easily identify the last successful merge commit in
             # the shadow repository.
             shadow_repo.set_refs('refs/heads/pr-merge', merge_commit_id)
             merge_ref = Reference('branch', 'pr-merge', merge_commit_id)
         except RepositoryError as e:
             log.exception('Failure when doing local merge on git shadow repo')
             if isinstance(e, UnresolvedFilesInRepo):
                 metadata['unresolved_files'] = '\n* conflict: ' + ('\n * conflict: '.join(e.args[0]))
 
             merge_possible = False
             merge_failure_reason = MergeFailureReason.MERGE_FAILED
 
         if merge_possible and not dry_run:
             try:
                 shadow_repo._local_push(
                     pr_branch, self.path, target_ref.name, enable_hooks=True,
                     rc_scm_data=self.config.get('rhodecode', 'RC_SCM_DATA'))
                 merge_succeeded = True
             except RepositoryError:
                 log.exception(
                     'Failure when doing local push from the shadow '
                     'repository to the target repository at %s.', self.path)
                 merge_succeeded = False
                 merge_failure_reason = MergeFailureReason.PUSH_FAILED
                 metadata['target'] = 'git shadow repo'
                 metadata['merge_commit'] = pr_branch
         else:
             merge_succeeded = False
 
         return MergeResponse(
             merge_possible, merge_succeeded, merge_ref, merge_failure_reason,
             metadata=metadata)