chore(code-cleanups): small fixes for readability

rhodecode/lib/vcs/backends/base.py

@@ -1,1984 +1,1983 b''
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Base module for all VCS systems
 """
 import os
 import re
 import time
 import shutil
 import datetime
 import fnmatch
 import itertools
 import logging
 import dataclasses
 import warnings
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 
 import rhodecode
 from rhodecode.translation import lazy_ugettext
 from rhodecode.lib.utils2 import safe_str, CachedProperty
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import (
     CommitError, EmptyRepositoryError, NodeAlreadyAddedError,
     NodeAlreadyChangedError, NodeAlreadyExistsError, NodeAlreadyRemovedError,
     NodeDoesNotExistError, NodeNotChangedError, VCSError,
     ImproperArchiveTypeError, BranchDoesNotExistError, CommitDoesNotExistError,
     RepositoryError)
 
 
 log = logging.getLogger(__name__)
 
 
 FILEMODE_DEFAULT = 0o100644
 FILEMODE_EXECUTABLE = 0o100755
 EMPTY_COMMIT_ID = '0' * 40
 
 
 @dataclasses.dataclass
 class Reference:
     type: str
     name: str
     commit_id: str
 
     def __iter__(self):
         yield self.type
         yield self.name
         yield self.commit_id
 
     @property
     def branch(self):
         if self.type == 'branch':
             return self.name
 
     @property
     def bookmark(self):
         if self.type == 'book':
             return self.name
 
     @property
     def to_str(self):
         return reference_to_unicode(self)
 
     def asdict(self):
         return dict(
             type=self.type,
             name=self.name,
             commit_id=self.commit_id
         )
 
 
 def unicode_to_reference(raw: str):
     """
     Convert a unicode (or string) to a reference object.
     If unicode evaluates to False it returns None.
     """
     if raw:
         refs = raw.split(':')
         return Reference(*refs)
     else:
         return None
 
 
 def reference_to_unicode(ref: Reference):
     """
     Convert a reference object to unicode.
     If reference is None it returns None.
     """
     if ref:
         return ':'.join(ref)
     else:
         return None
 
 
 class MergeFailureReason(object):
     """
     Enumeration with all the reasons why the server side merge could fail.
 
     DO NOT change the number of the reasons, as they may be stored in the
     database.
 
     Changing the name of a reason is acceptable and encouraged to deprecate old
     reasons.
     """
 
     # Everything went well.
     NONE = 0
 
     # An unexpected exception was raised. Check the logs for more details.
     UNKNOWN = 1
 
     # The merge was not successful, there are conflicts.
     MERGE_FAILED = 2
 
     # The merge succeeded but we could not push it to the target repository.
     PUSH_FAILED = 3
 
     # The specified target is not a head in the target repository.
     TARGET_IS_NOT_HEAD = 4
 
     # The source repository contains more branches than the target. Pushing
     # the merge will create additional branches in the target.
     HG_SOURCE_HAS_MORE_BRANCHES = 5
 
     # The target reference has multiple heads. That does not allow to correctly
     # identify the target location. This could only happen for mercurial
     # branches.
     HG_TARGET_HAS_MULTIPLE_HEADS = 6
 
     # The target repository is locked
     TARGET_IS_LOCKED = 7
 
     # Deprecated, use MISSING_TARGET_REF or MISSING_SOURCE_REF instead.
     # A involved commit could not be found.
     _DEPRECATED_MISSING_COMMIT = 8
 
     # The target repo reference is missing.
     MISSING_TARGET_REF = 9
 
     # The source repo reference is missing.
     MISSING_SOURCE_REF = 10
 
     # The merge was not successful, there are conflicts related to sub
     # repositories.
     SUBREPO_MERGE_FAILED = 11
 
 
 class UpdateFailureReason(object):
     """
     Enumeration with all the reasons why the pull request update could fail.
 
     DO NOT change the number of the reasons, as they may be stored in the
     database.
 
     Changing the name of a reason is acceptable and encouraged to deprecate old
     reasons.
     """
 
     # Everything went well.
     NONE = 0
 
     # An unexpected exception was raised. Check the logs for more details.
     UNKNOWN = 1
 
     # The pull request is up to date.
     NO_CHANGE = 2
 
     # The pull request has a reference type that is not supported for update.
     WRONG_REF_TYPE = 3
 
     # Update failed because the target reference is missing.
     MISSING_TARGET_REF = 4
 
     # Update failed because the source reference is missing.
     MISSING_SOURCE_REF = 5
 
 
 class MergeResponse(object):
 
     # uses .format(**metadata) for variables
     MERGE_STATUS_MESSAGES = {
         MergeFailureReason.NONE: lazy_ugettext(
             'This pull request can be automatically merged.'),
         MergeFailureReason.UNKNOWN: lazy_ugettext(
             'This pull request cannot be merged because of an unhandled exception. '
             '{exception}'),
         MergeFailureReason.MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of merge conflicts. {unresolved_files}'),
         MergeFailureReason.PUSH_FAILED: lazy_ugettext(
             'This pull request could not be merged because push to '
             'target:`{target}@{merge_commit}` failed.'),
         MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext(
             'This pull request cannot be merged because the target '
             '`{target_ref.name}` is not a head.'),
         MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext(
             'This pull request cannot be merged because the source contains '
             'more branches than the target.'),
         MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext(
             'This pull request cannot be merged because the target `{target_ref.name}` '
             'has multiple heads: `{heads}`.'),
         MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext(
             'This pull request cannot be merged because the target repository is '
             'locked by {locked_by}.'),
 
         MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be merged because the target '
             'reference `{target_ref.name}` is missing.'),
         MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be merged because the source '
             'reference `{source_ref.name}` is missing.'),
         MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of conflicts related '
             'to sub repositories.'),
 
         # Deprecations
         MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext(
             'This pull request cannot be merged because the target or the '
             'source reference is missing.'),
 
     }
 
     def __init__(self, possible, executed, merge_ref: Reference, failure_reason, metadata=None):
         self.possible = possible
         self.executed = executed
         self.merge_ref = merge_ref
         self.failure_reason = failure_reason
         self.metadata = metadata or {}
 
     def __repr__(self):
         return f'<MergeResponse:{self.label} {self.failure_reason}>'
 
     def __eq__(self, other):
         same_instance = isinstance(other, self.__class__)
         return same_instance \
                and self.possible == other.possible \
                and self.executed == other.executed \
                and self.failure_reason == other.failure_reason
 
     @property
     def label(self):
         label_dict = dict((v, k) for k, v in MergeFailureReason.__dict__.items() if
                           not k.startswith('_'))
         return label_dict.get(self.failure_reason)
 
     @property
     def merge_status_message(self):
         """
         Return a human friendly error message for the given merge status code.
         """
         msg = safe_str(self.MERGE_STATUS_MESSAGES[self.failure_reason])
 
         try:
             return msg.format(**self.metadata)
         except Exception:
             log.exception('Failed to format %s message', self)
             return msg
 
     def asdict(self):
         data = {}
         for k in ['possible', 'executed', 'merge_ref', 'failure_reason',
                   'merge_status_message']:
             data[k] = getattr(self, k)
         return data
 
 
 class TargetRefMissing(ValueError):
     pass
 
 
 class SourceRefMissing(ValueError):
     pass
 
 
 class BaseRepository(object):
     """
     Base Repository for final backends
 
     .. attribute:: DEFAULT_BRANCH_NAME
 
        name of default branch (i.e. "trunk" for svn, "master" for git etc.
 
     .. attribute:: commit_ids
 
        list of all available commit ids, in ascending order
 
     .. attribute:: path
 
        absolute path to the repository
 
     .. attribute:: bookmarks
 
        Mapping from name to :term:`Commit ID` of the bookmark. Empty in case
        there are no bookmarks or the backend implementation does not support
        bookmarks.
 
     .. attribute:: tags
 
        Mapping from name to :term:`Commit ID` of the tag.
 
     """
 
     DEFAULT_BRANCH_NAME = None
     DEFAULT_CONTACT = "Unknown"
     DEFAULT_DESCRIPTION = "unknown"
     EMPTY_COMMIT_ID = '0' * 40
     COMMIT_ID_PAT = re.compile(r'[0-9a-fA-F]{40}')
 
     path = None
 
     _is_empty = None
     _commit_ids = {}
 
     def __init__(self, repo_path, config=None, create=False, **kwargs):
         """
         Initializes repository. Raises RepositoryError if repository could
         not be find at the given ``repo_path`` or directory at ``repo_path``
         exists and ``create`` is set to True.
 
         :param repo_path: local path of the repository
         :param config: repository configuration
         :param create=False: if set to True, would try to create repository.
         :param src_url=None: if set, should be proper url from which repository
           would be cloned; requires ``create`` parameter to be set to True -
           raises RepositoryError if src_url is set and create evaluates to
           False
         """
         raise NotImplementedError
 
     def __repr__(self):
         return f'<{self.__class__.__name__} at {self.path}>'
 
     def __len__(self):
         return self.count()
 
     def __eq__(self, other):
         same_instance = isinstance(other, self.__class__)
         return same_instance and other.path == self.path
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     def get_create_shadow_cache_pr_path(self, db_repo):
         path = db_repo.cached_diffs_dir
         if not os.path.exists(path):
             os.makedirs(path, 0o755)
         return path
 
     @classmethod
     def get_default_config(cls, default=None):
         config = Config()
         if default and isinstance(default, list):
             for section, key, val in default:
                 config.set(section, key, val)
         return config
 
     @LazyProperty
     def _remote(self):
         raise NotImplementedError
 
     def _heads(self, branch=None):
         return []
 
     @LazyProperty
     def EMPTY_COMMIT(self):
         return EmptyCommit(self.EMPTY_COMMIT_ID)
 
     @LazyProperty
     def alias(self):
         for k, v in settings.BACKENDS.items():
             if v.split('.')[-1] == str(self.__class__.__name__):
                 return k
 
     @LazyProperty
     def name(self):
         return safe_str(os.path.basename(self.path))
 
     @LazyProperty
     def description(self):
         raise NotImplementedError
 
     def refs(self):
         """
         returns a `dict` with branches, bookmarks, tags, and closed_branches
         for this repository
         """
         return dict(
             branches=self.branches,
             branches_closed=self.branches_closed,
             tags=self.tags,
             bookmarks=self.bookmarks
         )
 
     @LazyProperty
     def branches(self):
         """
         A `dict` which maps branch names to commit ids.
         """
         raise NotImplementedError
 
     @LazyProperty
     def branches_closed(self):
         """
         A `dict` which maps tags names to commit ids.
         """
         raise NotImplementedError
 
     @LazyProperty
     def bookmarks(self):
         """
         A `dict` which maps tags names to commit ids.
         """
         raise NotImplementedError
 
     @LazyProperty
     def tags(self):
         """
         A `dict` which maps tags names to commit ids.
         """
         raise NotImplementedError
 
     @LazyProperty
     def size(self):
         """
         Returns combined size in bytes for all repository files
         """
         tip = self.get_commit()
         return tip.size
 
     def size_at_commit(self, commit_id):
         commit = self.get_commit(commit_id)
         return commit.size
 
     def _check_for_empty(self):
         no_commits = len(self._commit_ids) == 0
         if no_commits:
             # check on remote to be sure
             return self._remote.is_empty()
         else:
             return False
 
     def is_empty(self):
         if rhodecode.is_test:
             return self._check_for_empty()
 
         if self._is_empty is None:
             # cache empty for production, but not tests
             self._is_empty = self._check_for_empty()
 
         return self._is_empty
 
     @staticmethod
     def check_url(url, config):
         """
         Function will check given url and try to verify if it's a valid
         link.
         """
         raise NotImplementedError
 
     @staticmethod
     def is_valid_repository(path):
         """
         Check if given `path` contains a valid repository of this backend
         """
         raise NotImplementedError
 
     # ==========================================================================
     # COMMITS
     # ==========================================================================
 
     @CachedProperty
     def commit_ids(self):
         raise NotImplementedError
 
     def append_commit_id(self, commit_id):
         if commit_id not in self.commit_ids:
             self._rebuild_cache(self.commit_ids + [commit_id])
 
         # clear cache
         self._invalidate_prop_cache('commit_ids')
         self._is_empty = False
 
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None,
                    translate_tag=None, maybe_unreachable=False, reference_obj=None):
         """
         Returns instance of `BaseCommit` class. If `commit_id` and `commit_idx`
         are both None, most recent commit is returned.
 
         :param pre_load: Optional. List of commit attributes to load.
 
         :raises ``EmptyRepositoryError``: if there are no commits
         """
         raise NotImplementedError
 
     def __iter__(self):
         for commit_id in self.commit_ids:
             yield self.get_commit(commit_id=commit_id)
 
     def get_commits(
             self, start_id=None, end_id=None, start_date=None, end_date=None,
             branch_name=None, show_hidden=False, pre_load=None, translate_tags=None):
         """
         Returns iterator of `BaseCommit` objects from start to end
         not inclusive. This should behave just like a list, ie. end is not
         inclusive.
 
         :param start_id: None or str, must be a valid commit id
         :param end_id: None or str, must be a valid commit id
         :param start_date:
         :param end_date:
         :param branch_name:
         :param show_hidden:
         :param pre_load:
         :param translate_tags:
         """
         raise NotImplementedError
 
     def __getitem__(self, key):
         """
         Allows index based access to the commit objects of this repository.
         """
         pre_load = ["author", "branch", "date", "message", "parents"]
         if isinstance(key, slice):
             return self._get_range(key, pre_load)
         return self.get_commit(commit_idx=key, pre_load=pre_load)
 
     def _get_range(self, slice_obj, pre_load):
         for commit_id in self.commit_ids.__getitem__(slice_obj):
             yield self.get_commit(commit_id=commit_id, pre_load=pre_load)
 
     def count(self):
         return len(self.commit_ids)
 
     def tag(self, name, user, commit_id=None, message=None, date=None, **opts):
         """
         Creates and returns a tag for the given ``commit_id``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param commit_id: commit id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         raise NotImplementedError
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         raise NotImplementedError
 
     def get_diff(
             self, commit1, commit2, path=None, ignore_whitespace=False,
             context=3, path1=None):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         `commit2` since `commit1`.
 
         :param commit1: Entry point from which diff is shown. Can be
           ``self.EMPTY_COMMIT`` - in this case, patch showing all
           the changes since empty state of the repository until `commit2`
         :param commit2: Until which commit changes should be shown.
         :param path: Can be set to a path of a file to create a diff of that
           file. If `path1` is also set, this value is only associated to
           `commit2`.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         :param path1: Can be set to a path to associate with `commit1`. This
           parameter works only for backends which support diff generation for
           different paths. Other backends will raise a `ValueError` if `path1`
           is set and has a different value than `path`.
         :param file_path: filter this diff by given path pattern
         """
         raise NotImplementedError
 
     def strip(self, commit_id, branch=None):
         """
         Strip given commit_id from the repository
         """
         raise NotImplementedError
 
     def get_common_ancestor(self, commit_id1, commit_id2, repo2):
         """
         Return a latest common ancestor commit if one exists for this repo
         `commit_id1` vs `commit_id2` from `repo2`.
 
         :param commit_id1: Commit it from this repository to use as a
            target for the comparison.
         :param commit_id2: Source commit id to use for comparison.
         :param repo2: Source repository to use for comparison.
         """
         raise NotImplementedError
 
     def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None):
         """
         Compare this repository's revision `commit_id1` with `commit_id2`.
 
         Returns a tuple(commits, ancestor) that would be merged from
         `commit_id2`. Doing a normal compare (``merge=False``), ``None``
         will be returned as ancestor.
 
         :param commit_id1: Commit it from this repository to use as a
            target for the comparison.
         :param commit_id2: Source commit id to use for comparison.
         :param repo2: Source repository to use for comparison.
         :param merge: If set to ``True`` will do a merge compare which also
            returns the common ancestor.
         :param pre_load: Optional. List of commit attributes to load.
         """
         raise NotImplementedError
 
     def merge(self, repo_id, workspace_id, target_ref, source_repo, source_ref,
               user_name='', user_email='', message='', dry_run=False,
               use_rebase=False, close_branch=False):
         """
         Merge the revisions specified in `source_ref` from `source_repo`
         onto the `target_ref` of this repository.
 
         `source_ref` and `target_ref` are named tupls with the following
         fields `type`, `name` and `commit_id`.
 
         Returns a MergeResponse named tuple with the following fields
         'possible', 'executed', 'source_commit', 'target_commit',
         'merge_commit'.
 
         :param repo_id: `repo_id` target repo id.
         :param workspace_id: `workspace_id` unique identifier.
         :param target_ref: `target_ref` points to the commit on top of which
             the `source_ref` should be merged.
         :param source_repo: The repository that contains the commits to be
             merged.
         :param source_ref: `source_ref` points to the topmost commit from
             the `source_repo` which should be merged.
         :param user_name: Merge commit `user_name`.
         :param user_email: Merge commit `user_email`.
         :param message: Merge commit `message`.
         :param dry_run: If `True` the merge will not take place.
         :param use_rebase: If `True` commits from the source will be rebased
             on top of the target instead of being merged.
         :param close_branch: If `True` branch will be close before merging it
         """
         if dry_run:
             message = message or settings.MERGE_DRY_RUN_MESSAGE
             user_email = user_email or settings.MERGE_DRY_RUN_EMAIL
             user_name = user_name or settings.MERGE_DRY_RUN_USER
         else:
             if not user_name:
                 raise ValueError('user_name cannot be empty')
             if not user_email:
                 raise ValueError('user_email cannot be empty')
             if not message:
                 raise ValueError('message cannot be empty')
 
         try:
             return self._merge_repo(
                 repo_id, workspace_id, target_ref, source_repo,
                 source_ref, message, user_name, user_email, dry_run=dry_run,
                 use_rebase=use_rebase, close_branch=close_branch)
         except RepositoryError as exc:
             log.exception('Unexpected failure when running merge, dry-run=%s', dry_run)
             return MergeResponse(
                 False, False, None, MergeFailureReason.UNKNOWN,
                 metadata={'exception': str(exc)})
 
     def _merge_repo(self, repo_id, workspace_id, target_ref,
                     source_repo, source_ref, merge_message,
                     merger_name, merger_email, dry_run=False,
                     use_rebase=False, close_branch=False):
         """Internal implementation of merge."""
         raise NotImplementedError
 
     def _maybe_prepare_merge_workspace(
             self, repo_id, workspace_id, target_ref, source_ref):
         """
         Create the merge workspace.
 
         :param workspace_id: `workspace_id` unique identifier.
         """
         raise NotImplementedError
 
     @classmethod
     def _get_legacy_shadow_repository_path(cls, repo_path, workspace_id):
         """
         Legacy version that was used before. We still need it for
         backward compat
         """
         return os.path.join(
             os.path.dirname(repo_path),
             f'.__shadow_{os.path.basename(repo_path)}_{workspace_id}')
 
     @classmethod
     def _get_shadow_repository_path(cls, repo_path, repo_id, workspace_id):
         # The name of the shadow repository must start with '.', so it is
         # skipped by 'rhodecode.lib.utils.get_filesystem_repos'.
         legacy_repository_path = cls._get_legacy_shadow_repository_path(repo_path, workspace_id)
         if os.path.exists(legacy_repository_path):
             return legacy_repository_path
         else:
             return os.path.join(
                 os.path.dirname(repo_path),
                 f'.__shadow_repo_{repo_id}_{workspace_id}')
 
     def cleanup_merge_workspace(self, repo_id, workspace_id):
         """
         Remove merge workspace.
 
         This function MUST not fail in case there is no workspace associated to
         the given `workspace_id`.
 
         :param workspace_id: `workspace_id` unique identifier.
         """
         shadow_repository_path = self._get_shadow_repository_path(
             self.path, repo_id, workspace_id)
         shadow_repository_path_del = '{}.{}.delete'.format(
             shadow_repository_path, time.time())
 
         # move the shadow repo, so it never conflicts with the one used.
         # we use this method because shutil.rmtree had some edge case problems
         # removing symlinked repositories
         if not os.path.isdir(shadow_repository_path):
             return
 
         shutil.move(shadow_repository_path, shadow_repository_path_del)
         try:
             shutil.rmtree(shadow_repository_path_del, ignore_errors=False)
         except Exception:
             log.exception('Failed to gracefully remove shadow repo under %s',
                           shadow_repository_path_del)
             shutil.rmtree(shadow_repository_path_del, ignore_errors=True)
 
     # ========== #
     # COMMIT API #
     # ========== #
 
     @LazyProperty
     def in_memory_commit(self):
         """
         Returns :class:`InMemoryCommit` object for this repository.
         """
         raise NotImplementedError
 
     # ======================== #
     # UTILITIES FOR SUBCLASSES #
     # ======================== #
 
     def _validate_diff_commits(self, commit1, commit2):
         """
         Validates that the given commits are related to this repository.
 
         Intended as a utility for sub classes to have a consistent validation
         of input parameters in methods like :meth:`get_diff`.
         """
         self._validate_commit(commit1)
         self._validate_commit(commit2)
         if (isinstance(commit1, EmptyCommit) and
                 isinstance(commit2, EmptyCommit)):
             raise ValueError("Cannot compare two empty commits")
 
     def _validate_commit(self, commit):
         if not isinstance(commit, BaseCommit):
             raise TypeError(
                 "%s is not of type BaseCommit" % repr(commit))
         if commit.repository != self and not isinstance(commit, EmptyCommit):
             raise ValueError(
                 "Commit %s must be a valid commit from this repository %s, "
                 "related to this repository instead %s." %
                 (commit, self, commit.repository))
 
     def _validate_commit_id(self, commit_id):
         if not isinstance(commit_id, str):
             raise TypeError(f"commit_id must be a string value got {type(commit_id)} instead")
 
     def _validate_commit_idx(self, commit_idx):
         if not isinstance(commit_idx, int):
             raise TypeError(f"commit_idx must be a numeric value, got {type(commit_idx)}")
 
     def _validate_branch_name(self, branch_name):
         if branch_name and branch_name not in self.branches_all:
             msg = (f"Branch {branch_name} not found in {self}")
             raise BranchDoesNotExistError(msg)
 
     #
     # Supporting deprecated API parts
     # TODO: johbo: consider to move this into a mixin
     #
 
     @property
     def EMPTY_CHANGESET(self):
         warnings.warn(
             "Use EMPTY_COMMIT or EMPTY_COMMIT_ID instead", DeprecationWarning)
         return self.EMPTY_COMMIT_ID
 
     @property
     def revisions(self):
         warnings.warn("Use commits attribute instead", DeprecationWarning)
         return self.commit_ids
 
     @revisions.setter
     def revisions(self, value):
         warnings.warn("Use commits attribute instead", DeprecationWarning)
         self.commit_ids = value
 
     def get_changeset(self, revision=None, pre_load=None):
         warnings.warn("Use get_commit instead", DeprecationWarning)
         commit_id = None
         commit_idx = None
         if isinstance(revision, str):
             commit_id = revision
         else:
             commit_idx = revision
         return self.get_commit(
             commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load)
 
     def get_changesets(
             self, start=None, end=None, start_date=None, end_date=None,
             branch_name=None, pre_load=None):
         warnings.warn("Use get_commits instead", DeprecationWarning)
         start_id = self._revision_to_commit(start)
         end_id = self._revision_to_commit(end)
         return self.get_commits(
             start_id=start_id, end_id=end_id, start_date=start_date,
             end_date=end_date, branch_name=branch_name, pre_load=pre_load)
 
     def _revision_to_commit(self, revision):
         """
         Translates a revision to a commit_id
 
         Helps to support the old changeset based API which allows to use
         commit ids and commit indices interchangeable.
         """
         if revision is None:
             return revision
 
         if isinstance(revision, str):
             commit_id = revision
         else:
             commit_id = self.commit_ids[revision]
         return commit_id
 
     @property
     def in_memory_changeset(self):
         warnings.warn("Use in_memory_commit instead", DeprecationWarning)
         return self.in_memory_commit
 
     def get_path_permissions(self, username):
         """
         Returns a path permission checker or None if not supported
 
         :param username: session user name
         :return: an instance of BasePathPermissionChecker or None
         """
         return None
 
     def install_hooks(self, force=False):
         return self._remote.install_hooks(force)
 
     def get_hooks_info(self):
         return self._remote.get_hooks_info()
 
     def vcsserver_invalidate_cache(self, delete=False):
         return self._remote.vcsserver_invalidate_cache(delete)
 
 
 class BaseCommit(object):
     """
     Each backend should implement it's commit representation.
 
     **Attributes**
 
         ``repository``
             repository object within which commit exists
 
         ``id``
             The commit id, may be ``raw_id`` or i.e. for mercurial's tip
             just ``tip``.
 
         ``raw_id``
             raw commit representation (i.e. full 40 length sha for git
             backend)
 
         ``short_id``
             shortened (if apply) version of ``raw_id``; it would be simple
             shortcut for ``raw_id[:12]`` for git/mercurial backends or same
             as ``raw_id`` for subversion
 
         ``idx``
             commit index
 
         ``files``
             list of ``FileNode`` (``Node`` with NodeKind.FILE) objects
 
         ``dirs``
             list of ``DirNode`` (``Node`` with NodeKind.DIR) objects
 
         ``nodes``
             combined list of ``Node`` objects
 
         ``author``
             author of the commit, as unicode
 
         ``message``
             message of the commit, as unicode
 
         ``parents``
             list of parent commits
 
     """
     repository = None
     branch = None
 
     """
     Depending on the backend this should be set to the branch name of the
     commit. Backends not supporting branches on commits should leave this
     value as ``None``.
     """
 
     _ARCHIVE_PREFIX_TEMPLATE = '{repo_name}-{short_id}'
     """
     This template is used to generate a default prefix for repository archives
     if no prefix has been specified.
     """
 
     def __repr__(self):
         return self.__str__()
 
     def __str__(self):
         return f'<{self.__class__.__name__} at {self.idx}:{self.short_id}>'
 
     def __eq__(self, other):
         same_instance = isinstance(other, self.__class__)
         return same_instance and self.raw_id == other.raw_id
 
     def __json__(self):
         parents = []
         try:
             for parent in self.parents:
                 parents.append({'raw_id': parent.raw_id})
         except NotImplementedError:
             # empty commit doesn't have parents implemented
             pass
 
         return {
             'short_id': self.short_id,
             'raw_id': self.raw_id,
             'revision': self.idx,
             'message': self.message,
             'date': self.date,
             'author': self.author,
             'parents': parents,
             'branch': self.branch
         }
 
     def __getstate__(self):
         d = self.__dict__.copy()
         d.pop('_remote', None)
         d.pop('repository', None)
         return d
 
     def get_remote(self):
         return self._remote
 
     def serialize(self):
         return self.__json__()
 
     def _get_refs(self):
         return {
             'branches': [self.branch] if self.branch else [],
             'bookmarks': getattr(self, 'bookmarks', []),
             'tags': self.tags
         }
 
     @LazyProperty
     def last(self):
         """
         ``True`` if this is last commit in repository, ``False``
         otherwise; trying to access this attribute while there is no
         commits would raise `EmptyRepositoryError`
         """
         if self.repository is None:
             raise CommitError("Cannot check if it's most recent commit")
         return self.raw_id == self.repository.commit_ids[-1]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         raise NotImplementedError
 
     @LazyProperty
     def first_parent(self):
         """
         Returns list of parent commits.
         """
         return self.parents[0] if self.parents else EmptyCommit()
 
     @property
     def merge(self):
         """
         Returns boolean if commit is a merge.
         """
         return len(self.parents) > 1
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         raise NotImplementedError
 
     @LazyProperty
     def id(self):
         """
         Returns string identifying this commit.
         """
         raise NotImplementedError
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this commit.
         """
         raise NotImplementedError
 
     @LazyProperty
     def short_id(self):
         """
         Returns shortened version of ``raw_id`` attribute, as string,
         identifying this commit, useful for presentation to users.
         """
         raise NotImplementedError
 
     @LazyProperty
     def idx(self):
         """
         Returns integer identifying this commit.
         """
         raise NotImplementedError
 
     @LazyProperty
     def committer(self):
         """
         Returns committer for this commit
         """
         raise NotImplementedError
 
     @LazyProperty
     def committer_name(self):
         """
         Returns committer name for this commit
         """
 
         return author_name(self.committer)
 
     @LazyProperty
     def committer_email(self):
         """
         Returns committer email address for this commit
         """
 
         return author_email(self.committer)
 
     @LazyProperty
     def author(self):
         """
         Returns author for this commit
         """
 
         raise NotImplementedError
 
     @LazyProperty
     def author_name(self):
         """
         Returns author name for this commit
         """
 
         return author_name(self.author)
 
     @LazyProperty
     def author_email(self):
         """
         Returns author email address for this commit
         """
 
         return author_email(self.author)
 
     def get_file_mode(self, path: bytes):
         """
         Returns stat mode of the file at `path`.
         """
         raise NotImplementedError
 
     def is_link(self, path):
         """
         Returns ``True`` if given `path` is a symlink
         """
         raise NotImplementedError
 
     def is_node_binary(self, path):
         """
         Returns ``True`` is given path is a binary file
         """
         raise NotImplementedError
 
     def node_md5_hash(self, path):
         """
         Returns md5 hash of a node data
         """
         raise NotImplementedError
 
     def get_file_content(self, path) -> bytes:
         """
         Returns content of the file at the given `path`.
         """
         raise NotImplementedError
 
     def get_file_content_streamed(self, path):
         """
         returns a streaming response from vcsserver with file content
         """
         raise NotImplementedError
 
     def get_file_size(self, path):
         """
         Returns size of the file at the given `path`.
         """
         raise NotImplementedError
 
     def get_path_commit(self, path, pre_load=None):
         """
         Returns last commit of the file at the given `path`.
 
         :param pre_load: Optional. List of commit attributes to load.
         """
         commits = self.get_path_history(path, limit=1, pre_load=pre_load)
         if not commits:
             raise RepositoryError(
                 'Failed to fetch history for path {}. '
                 'Please check if such path exists in your repository'.format(
                     path))
         return commits[0]
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of :class:`BaseCommit`
         objects for which file at given `path` has been modified.
 
         :param limit: Optional. Allows to limit the size of the returned
            history. This is intended as a hint to the underlying backend, so
            that it can apply optimizations depending on the limit.
         :param pre_load: Optional. List of commit attributes to load.
         """
         raise NotImplementedError
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
         lineno, sha, commit lazy loader and line
 
         :param pre_load: Optional. List of commit attributes to load.
         """
         raise NotImplementedError
 
     def get_nodes(self, path, pre_load=None):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of commit at the given ``path``.
 
         :raises ``CommitError``: if node at the given ``path`` is not
           instance of ``DirNode``
         """
         raise NotImplementedError
 
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``.
 
         :raises ``NodeDoesNotExistError``: if there is no node at the given
           ``path``
         """
         raise NotImplementedError
 
     def get_largefile_node(self, path):
         """
         Returns the path to largefile from Mercurial/Git-lfs storage.
         or None if it's not a largefile node
         """
         return None
 
     def archive_repo(self, archive_name_key, kind='tgz', subrepos=None,
                      archive_dir_name=None, write_metadata=False, mtime=None,
                      archive_at_path='/', cache_config=None):
         """
         Creates an archive containing the contents of the repository.
 
         :param archive_name_key: unique key under this archive should be generated
         :param kind: one of the following: ``"tbz2"``, ``"tgz"``, ``"zip"``.
         :param archive_dir_name: name of root directory in archive.
             Default is repository name and commit's short_id joined with dash:
             ``"{repo_name}-{short_id}"``.
         :param write_metadata: write a metadata file into archive.
         :param mtime: custom modification time for archive creation, defaults
             to time.time() if not given.
         :param archive_at_path: pack files at this path (default '/')
         :param cache_config: config spec to send to vcsserver to configure the backend to store files
 
         :raise VCSError: If prefix has a problem.
         """
         cache_config = cache_config or {}
         allowed_kinds = [x[0] for x in settings.ARCHIVE_SPECS]
         if kind not in allowed_kinds:
             raise ImproperArchiveTypeError(
                 f'Archive kind ({kind}) not supported use one of {allowed_kinds}')
 
         archive_dir_name = self._validate_archive_prefix(archive_dir_name)
         mtime = mtime is not None or time.mktime(self.date.timetuple())
         commit_id = self.raw_id
 
         return self.repository._remote.archive_repo(
             archive_name_key, kind, mtime, archive_at_path,
             archive_dir_name, commit_id, cache_config)
 
     def _validate_archive_prefix(self, archive_dir_name):
         if archive_dir_name is None:
             archive_dir_name = self._ARCHIVE_PREFIX_TEMPLATE.format(
                 repo_name=safe_str(self.repository.name),
                 short_id=self.short_id)
         elif not isinstance(archive_dir_name, str):
             raise ValueError(f"archive_dir_name is not str object but: {type(archive_dir_name)}")
         elif archive_dir_name.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif archive_dir_name.strip() == '':
             raise VCSError("Prefix cannot be empty")
         elif not archive_dir_name.isascii():
             raise VCSError("Prefix cannot contain non ascii characters")
         return archive_dir_name
 
     @LazyProperty
     def root(self):
         """
         Returns ``RootNode`` object for this commit.
         """
         return self.get_node('')
 
     def next(self, branch=None):
         """
         Returns next commit from current, if branch is gives it will return
         next commit belonging to this branch
 
         :param branch: show commits within the given named branch
         """
         indexes = range(self.idx + 1, self.repository.count())
         return self._find_next(indexes, branch)
 
     def prev(self, branch=None):
         """
         Returns previous commit from current, if branch is gives it will
         return previous commit belonging to this branch
 
         :param branch: show commit within the given named branch
         """
         indexes = range(self.idx - 1, -1, -1)
         return self._find_next(indexes, branch)
 
     def _find_next(self, indexes, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on commit not belonging '
                            'to that branch')
 
         for next_idx in indexes:
             commit = self.repository.get_commit(commit_idx=next_idx)
             if branch and branch != commit.branch:
                 continue
             return commit
         raise CommitDoesNotExistError
 
     def diff(self, ignore_whitespace=True, context=3):
         """
         Returns a `Diff` object representing the change made by this commit.
         """
         parent = self.first_parent
         diff = self.repository.get_diff(
             parent, self,
             ignore_whitespace=ignore_whitespace,
             context=context)
         return diff
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def size(self):
         """
         Returns total number of bytes from contents of all filenodes.
         """
         return sum(node.size for node in self.get_filenodes_generator())
 
     def walk(self, topurl=''):
         """
         Similar to os.walk method. Insted of filesystem it walks through
         commit starting at given ``topurl``.  Returns generator of tuples
         (top_node, dirnodes, filenodes).
         """
         from rhodecode.lib.vcs.nodes import DirNode
 
         if isinstance(topurl, DirNode):
             top_node = topurl
         else:
             top_node = self.get_node(topurl)
 
         has_default_pre_load = False
         if isinstance(top_node, DirNode):
             # used to inject as we walk same defaults as given top_node
             default_pre_load = top_node.default_pre_load
             has_default_pre_load = True
 
         if not top_node.is_dir():
             return
         yield top_node, top_node.dirs, top_node.files
         for dir_node in top_node.dirs:
             if has_default_pre_load:
                 dir_node.default_pre_load = default_pre_load
             yield from self.walk(dir_node)
 
     def get_filenodes_generator(self):
         """
         Returns generator that yields *all* file nodes.
         """
         for topnode, dirs, files in self.walk():
             yield from files
 
     #
     # Utilities for sub classes to support consistent behavior
     #
 
     def no_node_at_path(self, path):
         return NodeDoesNotExistError(
             f"There is no file nor directory at the given path: "
             f"`{safe_str(path)}` at commit {self.short_id}")
 
     def _fix_path(self, path: str) -> str:
         """
         Paths are stored without trailing slash so we need to get rid off it if
         needed.
         """
         return safe_str(path).rstrip('/')
 
     #
     # Deprecated API based on changesets
     #
 
     @property
     def revision(self):
         warnings.warn("Use idx instead", DeprecationWarning)
         return self.idx
 
     @revision.setter
     def revision(self, value):
         warnings.warn("Use idx instead", DeprecationWarning)
         self.idx = value
 
     def get_file_changeset(self, path):
         warnings.warn("Use get_path_commit instead", DeprecationWarning)
         return self.get_path_commit(path)
 
 
 class BaseChangesetClass(type):
 
     def __instancecheck__(self, instance):
         return isinstance(instance, BaseCommit)
 
 
 class BaseChangeset(BaseCommit, metaclass=BaseChangesetClass):
 
     def __new__(cls, *args, **kwargs):
         warnings.warn(
             "Use BaseCommit instead of BaseChangeset", DeprecationWarning)
         return super().__new__(cls, *args, **kwargs)
 
 
 class BaseInMemoryCommit(object):
     """
     Represents differences between repository's state (most recent head) and
     changes made *in place*.
 
     **Attributes**
 
         ``repository``
             repository object for this in-memory-commit
 
         ``added``
             list of ``FileNode`` objects marked as *added*
 
         ``changed``
             list of ``FileNode`` objects marked as *changed*
 
         ``removed``
             list of ``FileNode`` or ``RemovedFileNode`` objects marked to be
             *removed*
 
         ``parents``
             list of :class:`BaseCommit` instances representing parents of
             in-memory commit. Should always be 2-element sequence.
 
     """
 
     def __init__(self, repository):
         self.repository = repository
         self.added = []
         self.changed = []
         self.removed = []
         self.parents = []
 
     def add(self, *filenodes):
         """
         Marks given ``FileNode`` objects as *to be committed*.
 
         :raises ``NodeAlreadyExistsError``: if node with same path exists at
            latest commit
         :raises ``NodeAlreadyAddedError``: if node with same path is already
            marked as *added*
         """
         # Check if not already marked as *added* first
         for node in filenodes:
             if node.path in (n.path for n in self.added):
                 raise NodeAlreadyAddedError(
                     "Such FileNode %s is already marked for addition"
                     % node.path)
         for node in filenodes:
             self.added.append(node)
 
     def change(self, *filenodes):
         """
         Marks given ``FileNode`` objects to be *changed* in next commit.
 
         :raises ``EmptyRepositoryError``: if there are no commits yet
         :raises ``NodeAlreadyExistsError``: if node with same path is already
            marked to be *changed*
         :raises ``NodeAlreadyRemovedError``: if node with same path is already
            marked to be *removed*
         :raises ``NodeDoesNotExistError``: if node doesn't exist in latest
            commit
         :raises ``NodeNotChangedError``: if node hasn't really be changed
         """
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError(
                     "Node at %s is already marked as removed" % node.path)
         try:
             self.repository.get_commit()
         except EmptyRepositoryError:
             raise EmptyRepositoryError(
                 "Nothing to change - try to *add* new nodes rather than "
                 "changing them")
         for node in filenodes:
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError(
                     "Node at '%s' is already marked as changed" % node.path)
             self.changed.append(node)
 
     def remove(self, *filenodes):
         """
         Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be
         *removed* in next commit.
 
         :raises ``NodeAlreadyRemovedError``: if node has been already marked to
            be *removed*
         :raises ``NodeAlreadyChangedError``: if node has been already marked to
            be *changed*
         """
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError(
                     "Node is already marked to for removal at %s" % node.path)
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError(
                     "Node is already marked to be changed at %s" % node.path)
             # We only mark node as *removed* - real removal is done by
             # commit method
             self.removed.append(node)
 
     def reset(self):
         """
         Resets this instance to initial state (cleans ``added``, ``changed``
         and ``removed`` lists).
         """
         self.added = []
         self.changed = []
         self.removed = []
         self.parents = []
 
     def get_ipaths(self):
         """
         Returns generator of paths from nodes marked as added, changed or
         removed.
         """
         for node in itertools.chain(self.added, self.changed, self.removed):
             yield node.path
 
     def get_paths(self):
         """
         Returns list of paths from nodes marked as added, changed or removed.
         """
         return list(self.get_ipaths())
 
     def check_integrity(self, parents=None):
         """
         Checks in-memory commit's integrity. Also, sets parents if not
         already set.
 
         :raises CommitError: if any error occurs (i.e.
           ``NodeDoesNotExistError``).
         """
         if not self.parents:
             parents = parents or []
             if len(parents) == 0:
                 try:
                     parents = [self.repository.get_commit(), None]
                 except EmptyRepositoryError:
                     parents = [None, None]
             elif len(parents) == 1:
                 parents += [None]
             self.parents = parents
 
         # Local parents, only if not None
         parents = [p for p in self.parents if p]
 
         # Check nodes marked as added
         for p in parents:
             for node in self.added:
                 try:
                     p.get_node(node.path)
                 except NodeDoesNotExistError:
                     pass
                 else:
                     raise NodeAlreadyExistsError(
                         f"Node `{node.path}` already exists at {p}")
 
         # Check nodes marked as changed
         missing = set(self.changed)
         not_changed = set(self.changed)
         if self.changed and not parents:
             raise NodeDoesNotExistError(str(self.changed[0].path))
         for p in parents:
             for node in self.changed:
                 try:
                     old = p.get_node(node.path)
                     missing.remove(node)
                     # if content actually changed, remove node from not_changed
                     if old.content != node.content:
                         not_changed.remove(node)
                 except NodeDoesNotExistError:
                     pass
         if self.changed and missing:
             raise NodeDoesNotExistError(
-                "Node `%s` marked as modified but missing in parents: %s"
-                % (node.path, parents))
+                f"Node `{node.path}` marked as modified but missing in parents: {parents}")
 
         if self.changed and not_changed:
             raise NodeNotChangedError(
                 "Node `%s` wasn't actually changed (parents: %s)"
                 % (not_changed.pop().path, parents))
 
         # Check nodes marked as removed
         if self.removed and not parents:
             raise NodeDoesNotExistError(
                 "Cannot remove node at %s as there "
                 "were no parents specified" % self.removed[0].path)
         really_removed = set()
         for p in parents:
             for node in self.removed:
                 try:
                     p.get_node(node.path)
                     really_removed.add(node)
                 except CommitError:
                     pass
         not_removed = set(self.removed) - really_removed
         if not_removed:
             # TODO: johbo: This code branch does not seem to be covered
             raise NodeDoesNotExistError(
                 "Cannot remove node at %s from "
                 "following parents: %s" % (not_removed, parents))
 
     def commit(self, message, author, parents=None, branch=None, date=None, **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created :class:`BaseCommit`. Updates repository's
         attribute `commits`.
 
         .. note::
 
            While overriding this method each backend's should call
            ``self.check_integrity(parents)`` in the first place.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
            would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
            ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
            branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         raise NotImplementedError
 
 
 class BaseInMemoryChangesetClass(type):
 
     def __instancecheck__(self, instance):
         return isinstance(instance, BaseInMemoryCommit)
 
 
 class BaseInMemoryChangeset(BaseInMemoryCommit, metaclass=BaseInMemoryChangesetClass):
 
     def __new__(cls, *args, **kwargs):
         warnings.warn(
             "Use BaseCommit instead of BaseInMemoryCommit", DeprecationWarning)
         return super().__new__(cls, *args, **kwargs)
 
 
 class EmptyCommit(BaseCommit):
     """
     An dummy empty commit. It's possible to pass hash when creating
     an EmptyCommit
     """
 
     def __init__(
             self, commit_id=EMPTY_COMMIT_ID, repo=None, alias=None, idx=-1,
             message='', author='', date=None):
         self._empty_commit_id = commit_id
         # TODO: johbo: Solve idx parameter, default value does not make
         # too much sense
         self.idx = idx
         self.message = message
         self.author = author
         self.date = date or datetime.datetime.fromtimestamp(0)
         self.repository = repo
         self.alias = alias
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this commit, useful for web
         representation.
         """
 
         return self._empty_commit_id
 
     @LazyProperty
     def branch(self):
         if self.alias:
             from rhodecode.lib.vcs.backends import get_backend
             return get_backend(self.alias).DEFAULT_BRANCH_NAME
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     def get_path_commit(self, path, pre_load=None):
         return self
 
     def get_file_content(self, path) -> bytes:
         return b''
 
     def get_file_content_streamed(self, path):
         yield self.get_file_content(path)
 
     def get_file_size(self, path):
         return 0
 
 
 class EmptyChangesetClass(type):
 
     def __instancecheck__(self, instance):
         return isinstance(instance, EmptyCommit)
 
 
 class EmptyChangeset(EmptyCommit, metaclass=EmptyChangesetClass):
 
     def __new__(cls, *args, **kwargs):
         warnings.warn(
             "Use EmptyCommit instead of EmptyChangeset", DeprecationWarning)
         return super(EmptyCommit, cls).__new__(cls, *args, **kwargs)
 
     def __init__(self, cs=EMPTY_COMMIT_ID, repo=None, requested_revision=None,
                  alias=None, revision=-1, message='', author='', date=None):
         if requested_revision is not None:
             warnings.warn(
                 "Parameter requested_revision not supported anymore",
                 DeprecationWarning)
         super().__init__(
             commit_id=cs, repo=repo, alias=alias, idx=revision,
             message=message, author=author, date=date)
 
         @property
         def revision(self):
             warnings.warn("Use idx instead", DeprecationWarning)
             return self.idx
 
         @revision.setter
         def revision(self, value):
             warnings.warn("Use idx instead", DeprecationWarning)
             self.idx = value
 
 
 class EmptyRepository(BaseRepository):
     def __init__(self, repo_path=None, config=None, create=False, **kwargs):
         pass
 
     def get_diff(self, *args, **kwargs):
         from rhodecode.lib.vcs.backends.git.diff import GitDiff
         return GitDiff(b'')
 
 
 class CollectionGenerator(object):
 
     def __init__(self, repo, commit_ids, collection_size=None, pre_load=None, translate_tag=None):
         self.repo = repo
         self.commit_ids = commit_ids
         self.collection_size = collection_size
         self.pre_load = pre_load
         self.translate_tag = translate_tag
 
     def __len__(self):
         if self.collection_size is not None:
             return self.collection_size
         return self.commit_ids.__len__()
 
     def __iter__(self):
         for commit_id in self.commit_ids:
             # TODO: johbo: Mercurial passes in commit indices or commit ids
             yield self._commit_factory(commit_id)
 
     def _commit_factory(self, commit_id):
         """
         Allows backends to override the way commits are generated.
         """
         return self.repo.get_commit(
             commit_id=commit_id, pre_load=self.pre_load,
             translate_tag=self.translate_tag)
 
     def __getitem__(self, key):
         """Return either a single element by index, or a sliced collection."""
 
         if isinstance(key, slice):
             commit_ids = self.commit_ids[key.start:key.stop]
 
         else:
             # single item
             commit_ids = self.commit_ids[key]
 
         return self.__class__(
             self.repo, commit_ids, pre_load=self.pre_load,
             translate_tag=self.translate_tag)
 
     def __repr__(self):
         return '<CollectionGenerator[len:%s]>' % (self.__len__())
 
 
 class Config(object):
     """
     Represents the configuration for a repository.
 
     The API is inspired by :class:`ConfigParser.ConfigParser` from the
     standard library. It implements only the needed subset.
     """
 
     def __init__(self):
         self._values = {}
 
     def copy(self):
         clone = Config()
         for section, values in self._values.items():
             clone._values[section] = values.copy()
         return clone
 
     def __repr__(self):
         return '<Config({} sections) at {}>'.format(
             len(self._values), hex(id(self)))
 
     def items(self, section):
         return self._values.get(section, {}).items()
 
     def get(self, section, option):
         return self._values.get(section, {}).get(option)
 
     def set(self, section, option, value):
         section_values = self._values.setdefault(section, {})
         section_values[option] = value
 
     def clear_section(self, section):
         self._values[section] = {}
 
     def serialize(self):
         """
         Creates a list of three tuples (section, key, value) representing
         this config object.
         """
         items = []
         for section in self._values:
             for option, value in self._values[section].items():
                 items.append(
                     (safe_str(section), safe_str(option), safe_str(value)))
         return items
 
 
 class Diff(object):
     """
     Represents a diff result from a repository backend.
 
     Subclasses have to provide a backend specific value for
     :attr:`_header_re` and :attr:`_meta_re`.
     """
     _meta_re = None
     _header_re: bytes = re.compile(br"")
 
     def __init__(self, raw_diff: bytes):
         if not isinstance(raw_diff, bytes):
             raise Exception(f'raw_diff must be bytes - got {type(raw_diff)}')
 
         self.raw = memoryview(raw_diff)
 
     def get_header_re(self):
         return self._header_re
 
     def chunks(self):
         """
         split the diff in chunks of separate --git a/file b/file chunks
         to make diffs consistent we must prepend with \n, and make sure
         we can detect last chunk as this was also has special rule
         """
 
         diff_parts = (b'\n' + bytes(self.raw)).split(b'\ndiff --git')
 
         chunks = diff_parts[1:]
         total_chunks = len(chunks)
 
         def diff_iter(_chunks):
             for cur_chunk, chunk in enumerate(_chunks, start=1):
                 yield DiffChunk(chunk, self, cur_chunk == total_chunks)
         return diff_iter(chunks)
 
 
 class DiffChunk(object):
 
     def __init__(self, chunk: bytes, diff_obj: Diff, is_last_chunk: bool):
         self.diff_obj = diff_obj
 
         # since we split by \ndiff --git that part is lost from original diff
         # we need to re-apply it at the end, EXCEPT ! if it's last chunk
         if not is_last_chunk:
             chunk += b'\n'
         header_re = self.diff_obj.get_header_re()
         match = header_re.match(chunk)
         self.header = match.groupdict()
         self.diff = chunk[match.end():]
         self.raw = chunk
 
     @property
     def header_as_str(self):
         if self.header:
             def safe_str_on_bytes(val):
                 if isinstance(val, bytes):
                     return safe_str(val)
                 return val
             return {safe_str(k): safe_str_on_bytes(v) for k, v in self.header.items()}
 
     def __repr__(self):
         return f'DiffChunk({self.header_as_str})'
 
 
 class BasePathPermissionChecker(object):
 
     @staticmethod
     def create_from_patterns(includes, excludes):
         if includes and '*' in includes and not excludes:
             return AllPathPermissionChecker()
         elif excludes and '*' in excludes:
             return NonePathPermissionChecker()
         else:
             return PatternPathPermissionChecker(includes, excludes)
 
     @property
     def has_full_access(self):
         raise NotImplementedError()
 
     def has_access(self, path):
         raise NotImplementedError()
 
 
 class AllPathPermissionChecker(BasePathPermissionChecker):
 
     @property
     def has_full_access(self):
         return True
 
     def has_access(self, path):
         return True
 
 
 class NonePathPermissionChecker(BasePathPermissionChecker):
 
     @property
     def has_full_access(self):
         return False
 
     def has_access(self, path):
         return False
 
 
 class PatternPathPermissionChecker(BasePathPermissionChecker):
 
     def __init__(self, includes, excludes):
         self.includes = includes
         self.excludes = excludes
         self.includes_re = [] if not includes else [
             re.compile(fnmatch.translate(pattern)) for pattern in includes]
         self.excludes_re = [] if not excludes else [
             re.compile(fnmatch.translate(pattern)) for pattern in excludes]
 
     @property
     def has_full_access(self):
         return '*' in self.includes and not self.excludes
 
     def has_access(self, path):
         for regex in self.excludes_re:
             if regex.match(path):
                 return False
         for regex in self.includes_re:
             if regex.match(path):
                 return True
         return False

rhodecode/model/pull_request.py

@@ -1,2392 +1,2390 b''
 # Copyright (C) 2012-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 pull request model for RhodeCode
 """
 
 import logging
 import os
 
 import datetime
 import urllib.request
 import urllib.parse
 import urllib.error
 import collections
 
 import dataclasses as dataclasses
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.translation import lazy_ugettext
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
 from rhodecode.lib.hash_utils import md5_safe
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import AttributeDict, get_current_rhodecode_user
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason,
     TargetRefMissing, SourceRefMissing)
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError)
 from rhodecode.model import BaseModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     aliased, null, lazyload, and_, or_, select, func, String, cast, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule, User)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
     EmailNotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 # Data structure to hold the response data when updating commits during a pull
 # request update.
 class UpdateResponse(object):
 
     def __init__(self, executed, reason, new, old, common_ancestor_id,
                  commit_changes, source_changed, target_changed):
 
         self.executed = executed
         self.reason = reason
         self.new = new
         self.old = old
         self.common_ancestor_id = common_ancestor_id
         self.changes = commit_changes
         self.source_changed = source_changed
         self.target_changed = target_changed
 
 
 def get_diff_info(
         source_repo, source_ref, target_repo, target_ref, get_authors=False,
         get_commit_authors=True):
     """
     Calculates detailed diff information for usage in preview of creation of a pull-request.
     This is also used for default reviewers logic
     """
 
     source_scm = source_repo.scm_instance()
     target_scm = target_repo.scm_instance()
 
     ancestor_id = target_scm.get_common_ancestor(target_ref, source_ref, source_scm)
     if not ancestor_id:
         raise ValueError(
             'cannot calculate diff info without a common ancestor. '
             'Make sure both repositories are related, and have a common forking commit.')
 
     # case here is that want a simple diff without incoming commits,
     # previewing what will be merged based only on commits in the source.
     log.debug('Using ancestor %s as source_ref instead of %s',
               ancestor_id, source_ref)
 
     # source of changes now is the common ancestor
     source_commit = source_scm.get_commit(commit_id=ancestor_id)
     # target commit becomes the source ref as it is the last commit
     # for diff generation this logic gives proper diff
     target_commit = source_scm.get_commit(commit_id=source_ref)
 
     vcs_diff = \
         source_scm.get_diff(commit1=source_commit, commit2=target_commit,
                             ignore_whitespace=False, context=3)
 
     diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
                                          diff_limit=0, file_limit=0, show_full_diff=True)
 
     _parsed = diff_processor.prepare()
 
     all_files = []
     all_files_changes = []
     changed_lines = {}
     stats = [0, 0]
     for f in _parsed:
         all_files.append(f['filename'])
         all_files_changes.append({
             'filename': f['filename'],
             'stats': f['stats']
         })
         stats[0] += f['stats']['added']
         stats[1] += f['stats']['deleted']
 
         changed_lines[f['filename']] = []
         if len(f['chunks']) < 2:
             continue
         # first line is "context" information
         for chunks in f['chunks'][1:]:
             for chunk in chunks['lines']:
                 if chunk['action'] not in ('del', 'mod'):
                     continue
                 changed_lines[f['filename']].append(chunk['old_lineno'])
 
     commit_authors = []
     user_counts = {}
     email_counts = {}
     author_counts = {}
     _commit_cache = {}
 
     commits = []
     if get_commit_authors:
         log.debug('Obtaining commit authors from set of commits')
         _compare_data = target_scm.compare(
             target_ref, source_ref, source_scm, merge=True,
             pre_load=["author", "date", "message"]
         )
 
         for commit in _compare_data:
             # NOTE(marcink): we serialize here, so we don't produce more vcsserver calls on data returned
             # at this function which is later called via JSON serialization
             serialized_commit = dict(
                 author=commit.author,
                 date=commit.date,
                 message=commit.message,
                 commit_id=commit.raw_id,
                 raw_id=commit.raw_id
             )
             commits.append(serialized_commit)
             user = User.get_from_cs_author(serialized_commit['author'])
             if user and user not in commit_authors:
                 commit_authors.append(user)
 
     # lines
     if get_authors:
         log.debug('Calculating authors of changed files')
         target_commit = source_repo.get_commit(ancestor_id)
 
         for fname, lines in changed_lines.items():
 
             try:
                 node = target_commit.get_node(fname, pre_load=["is_binary"])
             except Exception:
                 log.exception("Failed to load node with path %s", fname)
                 continue
 
             if not isinstance(node, FileNode):
                 continue
 
             # NOTE(marcink): for binary node we don't do annotation, just use last author
             if node.is_binary:
                 author = node.last_commit.author
                 email = node.last_commit.author_email
 
                 user = User.get_from_cs_author(author)
                 if user:
                     user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                 author_counts[author] = author_counts.get(author, 0) + 1
                 email_counts[email] = email_counts.get(email, 0) + 1
 
                 continue
 
             for annotation in node.annotate:
                 line_no, commit_id, get_commit_func, line_text = annotation
                 if line_no in lines:
                     if commit_id not in _commit_cache:
                         _commit_cache[commit_id] = get_commit_func()
                     commit = _commit_cache[commit_id]
                     author = commit.author
                     email = commit.author_email
                     user = User.get_from_cs_author(author)
                     if user:
                         user_counts[user.user_id] = user_counts.get(user.user_id, 0) + 1
                     author_counts[author] = author_counts.get(author, 0) + 1
                     email_counts[email] = email_counts.get(email, 0) + 1
 
     log.debug('Default reviewers processing finished')
 
     return {
         'commits': commits,
         'files': all_files_changes,
         'stats': stats,
         'ancestor': ancestor_id,
         # original authors of modified files
         'original_authors': {
             'users': user_counts,
             'authors': author_counts,
             'emails': email_counts,
         },
         'commit_authors': commit_authors
     }
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     DIFF_CONTEXT = diffs.DEFAULT_CONTEXT
 
     UPDATE_STATUS_MESSAGES = {
         UpdateFailureReason.NONE: lazy_ugettext(
             'Pull request update successful.'),
         UpdateFailureReason.UNKNOWN: lazy_ugettext(
             'Pull request update failed because of an unknown error.'),
         UpdateFailureReason.NO_CHANGE: lazy_ugettext(
             'No update needed because the source and target have not changed.'),
         UpdateFailureReason.WRONG_REF_TYPE: lazy_ugettext(
             'Pull request cannot be updated because the reference type is '
             'not supported for an update. Only Branch, Tag or Bookmark is allowed.'),
         UpdateFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be updated because the target '
             'reference is missing.'),
         UpdateFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
     REF_TYPES = ['bookmark', 'book', 'tag', 'branch']
     UPDATABLE_REF_TYPES = ['bookmark', 'book', 'branch']
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
             PullRequest, PullRequestVersion), pull_request)
 
     def _check_perms(self, perms, pull_request, user, api=False):
         if not api:
             return h.HasRepoPermissionAny(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
         else:
             return h.HasRepoPermissionAnyApi(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
 
     def check_user_read(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_merge(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'hg.admin',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_update(self, pull_request, user, api=False):
         owner = user.user_id == pull_request.user_id
         return self.check_user_merge(pull_request, user, api) or owner
 
     def check_user_delete(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         _perms = ('repository.admin',)
         return self._check_perms(_perms, pull_request, user) or owner
 
     def is_user_reviewer(self, pull_request, user):
         return user.user_id in [
             x.user_id for x in
             pull_request.get_pull_request_reviewers(PullRequestReviewers.ROLE_REVIEWER)
             if x.user
         ]
 
     def check_user_change_status(self, pull_request, user, api=False):
         return self.check_user_update(pull_request, user, api) \
                or self.is_user_reviewer(pull_request, user)
 
     def check_user_comment(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         return self.check_user_read(pull_request, user) or owner
 
     def get(self, pull_request):
         return self.__get_pull_request(pull_request)
 
     def _prepare_get_all_query(self, repo_name, search_q=None, source=False,
                                statuses=None, opened_by=None, order_by=None,
                                order_dir='desc', only_created=False):
         repo = None
         if repo_name:
             repo = self._get_repo(repo_name)
 
         q = PullRequest.query()
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         # source or target
         if repo and source:
             q = q.filter(PullRequest.source_repo == repo)
         elif repo:
             q = q.filter(PullRequest.target_repo == repo)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         # opened by filter
         if opened_by:
             q = q.filter(PullRequest.user_id.in_(opened_by))
 
         # only get those that are in "created" state
         if only_created:
             q = q.filter(PullRequest.pull_request_state == PullRequest.STATE_CREATED)
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'id': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_all(self, repo_name, search_q=None, source=False, statuses=None,
                   opened_by=None):
         """
         Count the number of pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by)
 
         return q.count()
 
     def get_all(self, repo_name, search_q=None, source=False, statuses=None,
                 opened_by=None, offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, search_q=search_q, source=source, statuses=statuses,
             opened_by=opened_by, order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def count_awaiting_review(self, repo_name, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_review(
             repo_name, search_q=search_q, statuses=statuses)
 
         return len(pull_requests)
 
     def get_awaiting_review(self, repo_name, search_q=None, statuses=None,
                             offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review.
 
         :param repo_name: target or source repo
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         _filtered_pull_requests = []
         for pr in pull_requests:
             status = pr.calculated_review_status()
             if status in [ChangesetStatus.STATUS_NOT_REVIEWED,
                           ChangesetStatus.STATUS_UNDER_REVIEW]:
                 _filtered_pull_requests.append(pr)
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def _prepare_awaiting_my_review_review_query(
             self, repo_name, user_id, search_q=None, statuses=None,
             order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
         repo_alias = aliased(Repository)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .join(repo_alias,
                   repo_alias.repo_id == pull_request_alias.target_repo_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(repo_alias.repo_name == repo_name) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :returns: int number of pull requests
         """
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses)
         return q.count()
 
     def get_awaiting_my_review(self, repo_name, user_id, search_q=None, statuses=None,
                                offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review from a specific user.
 
         :param repo_name: target or source repo
         :param user_id: reviewer user of the pull request
         :param search_q: filter by text
         :param statuses: list of pull request statuses
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
 
         q = self._prepare_awaiting_my_review_review_query(
             repo_name, user_id, search_q=search_q, statuses=statuses,
             order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_im_participating_query(self, user_id=None, statuses=None, query='',
                                         order_by=None, order_dir='desc'):
         """
         return a query of pull-requests user is an creator, or he's added as a reviewer
         """
         q = PullRequest.query()
         if user_id:
 
             base_query = select(PullRequestReviewers)\
                 .where(PullRequestReviewers.user_id == user_id)\
                 .with_only_columns(PullRequestReviewers.pull_request_id)
 
             user_filter = or_(
                 PullRequest.user_id == user_id,
                 PullRequest.pull_request_id.in_(base_query)
             )
             q = PullRequest.query().filter(user_filter)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == PullRequest.user_id)
             q = q.filter(or_(
                 cast(PullRequest.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 PullRequest.title.ilike(like_expression),
                 PullRequest.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': PullRequest.pull_request_id,
             'title': PullRequest.title,
             'updated_on_raw': PullRequest.updated_on,
             'target_repo': PullRequest.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in(self, user_id=None, statuses=None, query=''):
         q = self._prepare_im_participating_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in(
             self, user_id=None, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that i'm participating in as a reviewer, or i have opened
         """
 
         q = self._prepare_im_participating_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def _prepare_participating_in_for_review_query(
             self, user_id, statuses=None, query='', order_by=None, order_dir='desc'):
 
         for_review_statuses = [
             ChangesetStatus.STATUS_UNDER_REVIEW, ChangesetStatus.STATUS_NOT_REVIEWED
         ]
 
         pull_request_alias = aliased(PullRequest)
         status_alias = aliased(ChangesetStatus)
         reviewers_alias = aliased(PullRequestReviewers)
 
         last_ver_subq = Session()\
             .query(func.min(ChangesetStatus.version)) \
             .filter(ChangesetStatus.pull_request_id == reviewers_alias.pull_request_id)\
             .filter(ChangesetStatus.user_id == reviewers_alias.user_id) \
             .subquery()
 
         q = Session().query(pull_request_alias) \
             .options(lazyload(pull_request_alias.author)) \
             .join(reviewers_alias,
                   reviewers_alias.pull_request_id == pull_request_alias.pull_request_id) \
             .outerjoin(status_alias,
                        and_(status_alias.user_id == reviewers_alias.user_id,
                             status_alias.pull_request_id == reviewers_alias.pull_request_id)) \
             .filter(or_(status_alias.version == null(),
                         status_alias.version == last_ver_subq)) \
             .filter(reviewers_alias.user_id == user_id) \
             .filter(or_(status_alias.status == null(), status_alias.status.in_(for_review_statuses))) \
             .group_by(pull_request_alias)
 
         # closed,opened
         if statuses:
             q = q.filter(pull_request_alias.status.in_(statuses))
 
         if query:
             like_expression = u'%{}%'.format(safe_str(query))
             q = q.join(User, User.user_id == pull_request_alias.user_id)
             q = q.filter(or_(
                 cast(pull_request_alias.pull_request_id, String).ilike(like_expression),
                 User.username.ilike(like_expression),
                 pull_request_alias.title.ilike(like_expression),
                 pull_request_alias.description.ilike(like_expression),
             ))
 
         order_map = {
             'name_raw': pull_request_alias.pull_request_id,
             'title': pull_request_alias.title,
             'updated_on_raw': pull_request_alias.updated_on,
             'target_repo': pull_request_alias.target_repo_id
         }
         if order_by and order_by in order_map:
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in_for_review(self, user_id, statuses=None, query=''):
         q = self._prepare_participating_in_for_review_query(user_id, statuses=statuses, query=query)
         return q.count()
 
     def get_im_participating_in_for_review(
             self, user_id, statuses=None, query='', offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that needs user approval or rejection
         """
 
         q = self._prepare_participating_in_for_review_query(
             user_id, statuses=statuses, query=query, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def get_versions(self, pull_request):
         """
         returns version of pull request sorted by ID descending
         """
         return PullRequestVersion.query()\
             .filter(PullRequestVersion.pull_request == pull_request)\
             .order_by(PullRequestVersion.pull_request_version_id.asc())\
             .all()
 
     def get_pr_version(self, pull_request_id, version=None):
         at_version = None
 
         if version and version == 'latest':
             pull_request_ver = PullRequest.get(pull_request_id)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_obj
             at_version = 'latest'
         elif version:
             pull_request_ver = PullRequestVersion.get_or_404(version)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_ver.pull_request
             at_version = pull_request_ver.pull_request_version_id
         else:
             _org_pull_request_obj = pull_request_obj = PullRequest.get_or_404(
                 pull_request_id)
 
         pull_request_display_obj = PullRequest.get_pr_display_object(
             pull_request_obj, _org_pull_request_obj)
 
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
     def pr_commits_versions(self, versions):
         """
         Maps the pull-request commits into all known PR versions. This way we can obtain
         each pr version the commit was introduced in.
         """
         commit_versions = collections.defaultdict(list)
         num_versions = [x.pull_request_version_id for x in versions]
         for ver in versions:
             for commit_id in ver.revisions:
                 ver_idx = ChangesetComment.get_index_from_version(
                     ver.pull_request_version_id, num_versions=num_versions)
                 commit_versions[commit_id].append(ver_idx)
         return commit_versions
 
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, observers, title, description=None,
                common_ancestor_id=None,
                description_renderer=None,
                reviewer_data=None, translator=None, auth_user=None):
         translator = translator or get_current_request().translate
 
         created_by_user = self._get_user(created_by)
         auth_user = auth_user or created_by_user.AuthUser()
         source_repo = self._get_repo(source_repo)
         target_repo = self._get_repo(target_repo)
 
         pull_request = PullRequest()
         pull_request.source_repo = source_repo
         pull_request.source_ref = source_ref
         pull_request.target_repo = target_repo
         pull_request.target_ref = target_ref
         pull_request.revisions = revisions
         pull_request.title = title
         pull_request.description = description
         pull_request.description_renderer = description_renderer
         pull_request.author = created_by_user
         pull_request.reviewer_data = reviewer_data
         pull_request.pull_request_state = pull_request.STATE_CREATING
         pull_request.common_ancestor_id = common_ancestor_id
 
         Session().add(pull_request)
         Session().flush()
 
         reviewer_ids = set()
         # members / reviewers
         for reviewer_object in reviewers:
             user_id, reasons, mandatory, role, rules = reviewer_object
             user = self._get_user(user_id)
 
             # skip duplicates
             if user.user_id in reviewer_ids:
                 continue
 
             reviewer_ids.add(user.user_id)
 
             reviewer = PullRequestReviewers()
             reviewer.user = user
             reviewer.pull_request = pull_request
             reviewer.reasons = reasons
             reviewer.mandatory = mandatory
             reviewer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 review_group = rule.user_group_vote_rule(user_id)
                 # we check if this particular reviewer is member of a voting group
                 if review_group:
                     # NOTE(marcink):
                     # can be that user is member of more but we pick the first same,
                     # same as default reviewers algo
                     review_group = review_group[0]
 
                     rule_data = {
                         'rule_name':
                             rule.review_rule_name,
                         'rule_user_group_entry_id':
                             review_group.repo_review_rule_users_group_id,
                         'rule_user_group_name':
                             review_group.users_group.users_group_name,
                         'rule_user_group_members':
                             [x.user.username for x in review_group.users_group.members],
                         'rule_user_group_members_id':
                             [x.user.user_id for x in review_group.users_group.members],
                     }
                     # e.g {'vote_rule': -1, 'mandatory': True}
                     rule_data.update(review_group.rule_data())
 
                     reviewer.rule_data = rule_data
 
             Session().add(reviewer)
             Session().flush()
 
         for observer_object in observers:
             user_id, reasons, mandatory, role, rules = observer_object
             user = self._get_user(user_id)
 
             # skip duplicates from reviewers
             if user.user_id in reviewer_ids:
                 continue
 
             #reviewer_ids.add(user.user_id)
 
             observer = PullRequestReviewers()
             observer.user = user
             observer.pull_request = pull_request
             observer.reasons = reasons
             observer.mandatory = mandatory
             observer.role = role
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 # TODO(marcink): do we need this for observers ??
                 pass
 
             Session().add(observer)
             Session().flush()
 
         # Set approval status to "Under Review" for all commits which are
         # part of this pull request.
         ChangesetStatusModel().set_status(
             repo=target_repo,
             status=ChangesetStatus.STATUS_UNDER_REVIEW,
             user=created_by_user,
             pull_request=pull_request
         )
         # we commit early at this point. This has to do with a fact
         # that before queries do some row-locking. And because of that
         # we need to commit and finish transaction before below validate call
         # that for large repos could be long resulting in long row locks
         Session().commit()
 
         # prepare workspace, and run initial merge simulation. Set state during that
         # operation
         pull_request = PullRequest.get(pull_request.pull_request_id)
 
         # set as merging, for merge simulation, and if finished to created so we mark
         # simulation is working fine
         with pull_request.set_state(PullRequest.STATE_MERGING,
                                     final_state=PullRequest.STATE_CREATED) as state_obj:
             MergeCheck.validate(
                 pull_request, auth_user=auth_user, translator=translator)
 
         self.notify_reviewers(pull_request, reviewer_ids, created_by_user)
         self.trigger_pull_request_hook(pull_request, created_by_user, 'create')
 
         creation_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.create', {'data': creation_data},
             auth_user, pull_request)
 
         return pull_request
 
     def trigger_pull_request_hook(self, pull_request, user, action, data=None):
         pull_request = self.__get_pull_request(pull_request)
         target_scm = pull_request.target_repo.scm_instance()
         if action == 'create':
             trigger_hook = hooks_utils.trigger_create_pull_request_hook
         elif action == 'merge':
             trigger_hook = hooks_utils.trigger_merge_pull_request_hook
         elif action == 'close':
             trigger_hook = hooks_utils.trigger_close_pull_request_hook
         elif action == 'review_status_change':
             trigger_hook = hooks_utils.trigger_review_pull_request_hook
         elif action == 'update':
             trigger_hook = hooks_utils.trigger_update_pull_request_hook
         elif action == 'comment':
             trigger_hook = hooks_utils.trigger_comment_pull_request_hook
         elif action == 'comment_edit':
             trigger_hook = hooks_utils.trigger_comment_pull_request_edit_hook
         else:
             return
 
         log.debug('Handling pull_request %s trigger_pull_request_hook with action %s and hook: %s',
                   pull_request, action, trigger_hook)
         trigger_hook(
             username=user.username,
             repo_name=pull_request.target_repo.repo_name,
             repo_type=target_scm.alias,
             pull_request=pull_request,
             data=data)
 
     def _get_commit_ids(self, pull_request):
         """
         Return the commit ids of the merged pull request.
 
         This method is not dealing correctly yet with the lack of autoupdates
         nor with the implicit target updates.
         For example: if a commit in the source repo is already in the target it
         will be reported anyways.
         """
         merge_rev = pull_request.merge_rev
         if merge_rev is None:
             raise ValueError('This pull request was not merged yet')
 
         commit_ids = list(pull_request.revisions)
         if merge_rev not in commit_ids:
             commit_ids.append(merge_rev)
 
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
         repo_type = pull_request.source_repo.repo_type
         log.debug("Merging pull request %s", pull_request)
 
         extras['user_agent'] = '{}/internal-merge'.format(repo_type)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug("Merge was successful, updating the pull request comments.")
             self._comment_and_close_pr(pull_request, user, merge_state)
 
             self._log_audit_action(
                 'repo.pull_request.merge',
                 {'merge_state': merge_state.__dict__},
                 user, pull_request)
 
         else:
             log.warning("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
 
         message = safe_str(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
             pr_id=pull_request.pull_request_id,
             pr_title=pull_request.title,
             pr_desc=pull_request.description,
             source_repo=source_vcs.name,
             source_ref_name=pull_request.source_ref_parts.name,
             target_repo=target_vcs.name,
             target_ref_name=pull_request.target_ref_parts.name,
         )
 
         workspace_id = self._workspace_id(pull_request)
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         user_name = self._user_name_for_merging(pull_request, user)
 
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override
             # for a single call.
             target_vcs.config.set(
                 'rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
             merge_state = target_vcs.merge(
                 repo_id, workspace_id, target_ref, source_vcs,
                 pull_request.source_ref_parts,
                 user_name=user_name, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
 
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
         pull_request.merge_rev = merge_state.merge_ref.commit_id
         pull_request.updated_on = datetime.datetime.now()
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
             text=safe_str(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             closing_pr=True
         )
 
         Session().add(pull_request)
         Session().flush()
         # TODO: paris: replace invalidation with less radical solution
         ScmModel().mark_for_invalidation(
             pull_request.target_repo.repo_name)
         self.trigger_pull_request_hook(pull_request, user, 'merge')
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
         return source_ref_type in self.REF_TYPES
 
     def get_flow_commits(self, pull_request):
 
         # source repo
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_id = pull_request.source_ref_parts.commit_id
         source_repo = pull_request.source_repo.scm_instance()
 
         try:
             if source_ref_type in self.REF_TYPES:
                 source_commit = source_repo.get_commit(
                     source_ref_name, reference_obj=pull_request.source_ref_parts)
             else:
                 source_commit = source_repo.get_commit(source_ref_id)
         except CommitDoesNotExistError:
             raise SourceRefMissing()
 
         # target repo
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_id = pull_request.target_ref_parts.commit_id
         target_repo = pull_request.target_repo.scm_instance()
 
         try:
             if target_ref_type in self.REF_TYPES:
                 target_commit = target_repo.get_commit(
                     target_ref_name, reference_obj=pull_request.target_ref_parts)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
         except CommitDoesNotExistError:
             raise TargetRefMissing()
 
         return source_commit, target_commit
 
     def update_commits(self, pull_request, updating_user):
         """
         Get the updated list of commits for the pull request
         and return the new pull request version and the list
         of commits processed by this update action
 
         updating_user is the user_object who triggered the update
         """
         pull_request = self.__get_pull_request(pull_request)
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_id = pull_request.source_ref_parts.commit_id
 
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         if not self.has_valid_update_type(pull_request):
             log.debug("Skipping update of pull request %s due to ref type: %s",
                       pull_request, source_ref_type)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.WRONG_REF_TYPE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         try:
             source_commit, target_commit = self.get_flow_commits(pull_request)
         except SourceRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_SOURCE_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
         except TargetRefMissing:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=False, target_changed=False)
 
         source_changed = source_ref_id != source_commit.raw_id
         target_changed = target_ref_id != target_commit.raw_id
 
         if not (source_changed or target_changed):
             log.debug("Nothing changed in pull request %s", pull_request)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.NO_CHANGE,
                 old=pull_request, new=None, common_ancestor_id=None, commit_changes=None,
                 source_changed=target_changed, target_changed=source_changed)
 
         change_in_found = 'target repo' if target_changed else 'source repo'
         log.debug('Updating pull request because of change in %s detected',
                   change_in_found)
 
         # Finally there is a need for an update, in case of source change
         # we create a new version, else just an update
         if source_changed:
             pull_request_version = self._create_version_from_snapshot(pull_request)
             self._link_comments_to_version(pull_request_version)
         else:
             try:
                 ver = pull_request.versions[-1]
             except IndexError:
                 ver = None
 
             pull_request.pull_request_version_id = \
                 ver.pull_request_version_id if ver else None
             pull_request_version = pull_request
 
         source_repo = pull_request.source_repo.scm_instance()
         target_repo = pull_request.target_repo.scm_instance()
 
         # re-compute commit ids
         old_commit_ids = pull_request.revisions
         pre_load = ["author", "date", "message", "branch"]
         commit_ranges = target_repo.compare(
             target_commit.raw_id, source_commit.raw_id, source_repo, merge=True,
             pre_load=pre_load)
 
         target_ref = target_commit.raw_id
         source_ref = source_commit.raw_id
         ancestor_commit_id = target_repo.get_common_ancestor(
             target_ref, source_ref, source_repo)
 
         if not ancestor_commit_id:
             raise ValueError(
                 'cannot calculate diff info without a common ancestor. '
                 'Make sure both repositories are related, and have a common forking commit.')
 
         pull_request.common_ancestor_id = ancestor_commit_id
 
-        pull_request.source_ref = '%s:%s:%s' % (
-            source_ref_type, source_ref_name, source_commit.raw_id)
-        pull_request.target_ref = '%s:%s:%s' % (
-            target_ref_type, target_ref_name, ancestor_commit_id)
+        pull_request.source_ref = f'{source_ref_type}:{source_ref_name}:{source_commit.raw_id}'
+        pull_request.target_ref = f'{target_ref_type}:{target_ref_name}:{ancestor_commit_id}'
 
         pull_request.revisions = [
             commit.raw_id for commit in reversed(commit_ranges)]
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         new_commit_ids = pull_request.revisions
 
         old_diff_data, new_diff_data = self._generate_update_diffs(
             pull_request, pull_request_version)
 
         # calculate commit and file changes
         commit_changes = self._calculate_commit_id_changes(
             old_commit_ids, new_commit_ids)
         file_changes = self._calculate_file_changes(
             old_diff_data, new_diff_data)
 
         # set comments as outdated if DIFFS changed
         CommentsModel().outdate_comments(
             pull_request, old_diff_data=old_diff_data,
             new_diff_data=new_diff_data)
 
         valid_commit_changes = (commit_changes.added or commit_changes.removed)
         file_node_changes = (
             file_changes.added or file_changes.modified or file_changes.removed)
         pr_has_changes = valid_commit_changes or file_node_changes
 
         # Add an automatic comment to the pull request, in case
         # anything has changed
         if pr_has_changes:
             update_comment = CommentsModel().create(
                 text=self._render_update_message(ancestor_commit_id, commit_changes, file_changes),
                 repo=pull_request.target_repo,
                 user=pull_request.author,
                 pull_request=pull_request,
                 send_email=False, renderer=DEFAULT_COMMENTS_RENDERER)
 
             # Update status to "Under Review" for added commits
             for commit_id in commit_changes.added:
                 ChangesetStatusModel().set_status(
                     repo=pull_request.source_repo,
                     status=ChangesetStatus.STATUS_UNDER_REVIEW,
                     comment=update_comment,
                     user=pull_request.author,
                     pull_request=pull_request,
                     revision=commit_id)
 
         # initial commit
         Session().commit()
 
         if pr_has_changes:
             # send update email to users
             try:
                 self.notify_users(pull_request=pull_request, updating_user=updating_user,
                                   ancestor_commit_id=ancestor_commit_id,
                                   commit_changes=commit_changes,
                                   file_changes=file_changes)
                 Session().commit()
             except Exception:
                 log.exception('Failed to send email notification to users')
                 Session().rollback()
 
         log.debug(
             'Updated pull request %s, added_ids: %s, common_ids: %s, '
             'removed_ids: %s', pull_request.pull_request_id,
             commit_changes.added, commit_changes.common, commit_changes.removed)
         log.debug(
             'Updated pull request with the following file changes: %s',
             file_changes)
 
         log.info(
             "Updated pull request %s from commit %s to commit %s, "
             "stored new version %s of this pull request.",
             pull_request.pull_request_id, source_ref_id,
             pull_request.source_ref_parts.commit_id,
             pull_request_version.pull_request_version_id)
 
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'update')
 
         return UpdateResponse(
             executed=True, reason=UpdateFailureReason.NONE,
             old=pull_request, new=pull_request_version,
             common_ancestor_id=ancestor_commit_id, commit_changes=commit_changes,
             source_changed=source_changed, target_changed=target_changed)
 
     def _create_version_from_snapshot(self, pull_request):
         version = PullRequestVersion()
         version.title = pull_request.title
         version.description = pull_request.description
         version.status = pull_request.status
         version.pull_request_state = pull_request.pull_request_state
         version.created_on = datetime.datetime.now()
         version.updated_on = pull_request.updated_on
         version.user_id = pull_request.user_id
         version.source_repo = pull_request.source_repo
         version.source_ref = pull_request.source_ref
         version.target_repo = pull_request.target_repo
         version.target_ref = pull_request.target_ref
 
         version._last_merge_source_rev = pull_request._last_merge_source_rev
         version._last_merge_target_rev = pull_request._last_merge_target_rev
         version.last_merge_status = pull_request.last_merge_status
         version.last_merge_metadata = pull_request.last_merge_metadata
         version.shadow_merge_ref = pull_request.shadow_merge_ref
         version.merge_rev = pull_request.merge_rev
         version.reviewer_data = pull_request.reviewer_data
 
         version.revisions = pull_request.revisions
         version.common_ancestor_id = pull_request.common_ancestor_id
         version.pull_request = pull_request
         Session().add(version)
         Session().flush()
 
         return version
 
     def _generate_update_diffs(self, pull_request, pull_request_version):
 
         diff_context = (
             self.DIFF_CONTEXT +
             CommentsModel.needed_extra_diff_context())
         hide_whitespace_changes = False
         source_repo = pull_request_version.source_repo
         source_ref_id = pull_request_version.source_ref_parts.commit_id
         target_ref_id = pull_request_version.target_ref_parts.commit_id
         old_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         new_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         # NOTE: this was using diff_format='gitdiff'
         old_diff_data = diffs.DiffProcessor(old_diff, diff_format='newdiff')
         old_diff_data.prepare()
         new_diff_data = diffs.DiffProcessor(new_diff, diff_format='newdiff')
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
 
     def _link_comments_to_version(self, pull_request_version):
         """
         Link all unlinked comments of this pull request to the given version.
 
         :param pull_request_version: The `PullRequestVersion` to which
             the comments shall be linked.
 
         """
         pull_request = pull_request_version.pull_request
         comments = ChangesetComment.query()\
             .filter(
                 # TODO: johbo: Should we query for the repo at all here?
                 # Pending decision on how comments of PRs are to be related
                 # to either the source repo, the target repo or no repo at all.
                 ChangesetComment.repo_id == pull_request.target_repo.repo_id,
                 ChangesetComment.pull_request == pull_request,
                 ChangesetComment.pull_request_version == null())\
             .order_by(ChangesetComment.comment_id.asc())
 
         # TODO: johbo: Find out why this breaks if it is done in a bulk
         # operation.
         for comment in comments:
             comment.pull_request_version_id = (
                 pull_request_version.pull_request_version_id)
             Session().add(comment)
 
     def _calculate_commit_id_changes(self, old_ids, new_ids):
         added = [x for x in new_ids if x not in old_ids]
         common = [x for x in new_ids if x in old_ids]
         removed = [x for x in old_ids if x not in new_ids]
         total = new_ids
         return ChangeTuple(added, common, removed, total)
 
     def _calculate_file_changes(self, old_diff_data, new_diff_data):
 
         old_files = OrderedDict()
         for diff_data in old_diff_data.parsed_diff:
             old_files[diff_data['filename']] = md5_safe(diff_data['raw_diff'])
 
         added_files = []
         modified_files = []
         removed_files = []
         for diff_data in new_diff_data.parsed_diff:
             new_filename = diff_data['filename']
             new_hash = md5_safe(diff_data['raw_diff'])
 
             old_hash = old_files.get(new_filename)
             if not old_hash:
                 # file is not present in old diff, we have to figure out from parsed diff
                 # operation ADD/REMOVE
                 operations_dict = diff_data['stats']['ops']
                 if diffs.DEL_FILENODE in operations_dict:
                     removed_files.append(new_filename)
                 else:
                     added_files.append(new_filename)
             else:
                 if new_hash != old_hash:
                     modified_files.append(new_filename)
                 # now remove a file from old, since we have seen it already
                 del old_files[new_filename]
 
         # removed files is when there are present in old, but not in NEW,
         # since we remove old files that are present in new diff, left-overs
         # if any should be the removed files
         removed_files.extend(old_files.keys())
 
         return FileChangeTuple(added_files, modified_files, removed_files)
 
     def _render_update_message(self, ancestor_commit_id, changes, file_changes):
         """
         render the message using DEFAULT_COMMENTS_RENDERER (RST renderer),
         so it's always looking the same disregarding on which default
         renderer system is using.
 
         :param ancestor_commit_id: ancestor raw_id
         :param changes: changes named tuple
         :param file_changes: file changes named tuple
 
         """
         new_status = ChangesetStatus.get_status_lbl(
             ChangesetStatus.STATUS_UNDER_REVIEW)
 
         changed_files = (
             file_changes.added + file_changes.modified + file_changes.removed)
 
         params = {
             'under_review_label': new_status,
             'added_commits': changes.added,
             'removed_commits': changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'ancestor_commit_id': ancestor_commit_id
         }
         renderer = RstTemplateRenderer()
         return renderer.render('pull_request_update.mako', **params)
 
     def edit(self, pull_request, title, description, description_renderer, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
         if title:
             pull_request.title = title
         pull_request.description = description
         pull_request.updated_on = datetime.datetime.now()
         pull_request.description_renderer = description_renderer
         Session().add(pull_request)
         self._log_audit_action(
             'repo.pull_request.edit', {'old_data': old_data},
             user, pull_request)
 
     def update_reviewers(self, pull_request, reviewer_data, user):
         """
         Update the reviewers in the pull request
 
         :param pull_request: the pr to update
         :param reviewer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
 
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         reviewers = {}
         for user_id, reasons, mandatory, role, rules in reviewer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             reviewers[user_id] = {
                 'reasons': reasons, 'mandatory': mandatory, 'role': role}
 
         reviewers_ids = set(reviewers.keys())
         current_reviewers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_REVIEWER)
 
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         ids_to_add = reviewers_ids.difference(current_reviewers_ids)
         ids_to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers", ids_to_add)
         log.debug("Removing %s reviewers", ids_to_remove)
         changed = False
         added_audit_reviewers = []
         removed_audit_reviewers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             reviewer = PullRequestReviewers()
             reviewer.user = _usr
             reviewer.pull_request = pull_request
             reviewer.reasons = reviewers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # reviewer.mandatory = reviewers[uid]['reasons']
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             reviewer.role = PullRequestReviewers.ROLE_REVIEWER
             Session().add(reviewer)
             added_audit_reviewers.append(reviewer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             reviewers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_REVIEWER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in reviewers:
                 added_audit_reviewers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_reviewers:
             self._log_audit_action(
                 'repo.pull_request.reviewer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_reviewers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def update_observers(self, pull_request, observer_data, user):
         """
         Update the observers in the pull request
 
         :param pull_request: the pr to update
         :param observer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, role, [rules])]
         :param user: current use who triggers this action
         """
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         observers = {}
         for user_id, reasons, mandatory, role, rules in observer_data:
             if isinstance(user_id, (int, str)):
                 user_id = self._get_user(user_id).user_id
             observers[user_id] = {
                 'reasons': reasons, 'observers': mandatory, 'role': role}
 
         observers_ids = set(observers.keys())
         current_observers = PullRequestReviewers.get_pull_request_reviewers(
             pull_request.pull_request_id, role=PullRequestReviewers.ROLE_OBSERVER)
 
         current_observers_ids = set([x.user.user_id for x in current_observers])
 
         ids_to_add = observers_ids.difference(current_observers_ids)
         ids_to_remove = current_observers_ids.difference(observers_ids)
 
         log.debug("Adding %s observer", ids_to_add)
         log.debug("Removing %s observer", ids_to_remove)
         changed = False
         added_audit_observers = []
         removed_audit_observers = []
 
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             observer = PullRequestReviewers()
             observer.user = _usr
             observer.pull_request = pull_request
             observer.reasons = observers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # observer.mandatory = observer[uid]['reasons']
 
             # NOTE(marcink): role should be hardcoded, so we won't edit it.
             observer.role = PullRequestReviewers.ROLE_OBSERVER
             Session().add(observer)
             added_audit_observers.append(observer.get_dict())
 
         for uid in ids_to_remove:
             changed = True
             # NOTE(marcink): we fetch "ALL" reviewers objects using .all().
             # This is an edge case that handles previous state of having the same reviewer twice.
             # this CAN happen due to the lack of DB checks
             observers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.role == PullRequestReviewers.ROLE_OBSERVER,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
 
             for obj in observers:
                 added_audit_observers.append(obj.get_dict())
                 Session().delete(obj)
 
         if changed:
             Session().expire_all()
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         # finally store audit logs
         for user_data in added_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.add', {'data': user_data},
                 user, pull_request)
         for user_data in removed_audit_observers:
             self._log_audit_action(
                 'repo.pull_request.observer.delete', {'old_data': user_data},
                 user, pull_request)
 
         self.notify_observers(pull_request, ids_to_add, user)
         return ids_to_add, ids_to_remove
 
     def get_url(self, pull_request, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if permalink:
             return request.route_url(
                 'pull_requests_global',
                 pull_request_id=pull_request.pull_request_id,)
         else:
             return request.route_url('pullrequest_show',
                 repo_name=safe_str(pull_request.target_repo.repo_name),
                 pull_request_id=pull_request.pull_request_id,)
 
     def get_shadow_clone_url(self, pull_request, request=None):
         """
         Returns qualified url pointing to the shadow repository. If this pull
         request is closed there is no shadow repository and ``None`` will be
         returned.
         """
         if pull_request.is_closed():
             return None
         else:
             pr_url = urllib.parse.unquote(self.get_url(pull_request, request=request))
             return safe_str('{pr_url}/repository'.format(pr_url=pr_url))
 
     def _notify_reviewers(self, pull_request, user_ids, role, user):
         # notification to reviewers/observers
         if not user_ids:
             return
 
         log.debug('Notify following %s users about pull-request %s', role, user_ids)
 
         pull_request_obj = pull_request
         # get the current participants of this pull request
         recipients = user_ids
         notification_type = EmailNotificationModel.TYPE_PULL_REQUEST
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         # pull request specifics
         pull_request_commits = [
             (x.raw_id, x.message)
             for x in map(pr_source_repo.get_commit, pull_request.revisions)]
 
         current_rhodecode_user = user
         kwargs = {
             'user': current_rhodecode_user,
             'pull_request_author': pull_request.author,
             'pull_request': pull_request_obj,
             'pull_request_commits': pull_request_commits,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
             'thread_ids': [pr_url],
             'user_role': role
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=current_rhodecode_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=notification_type,
             recipients=recipients,
             email_kwargs=kwargs,
         )
 
     def notify_reviewers(self, pull_request, reviewers_ids, user):
         return self._notify_reviewers(pull_request, reviewers_ids,
                                       PullRequestReviewers.ROLE_REVIEWER, user)
 
     def notify_observers(self, pull_request, observers_ids, user):
         return self._notify_reviewers(pull_request, observers_ids,
                                       PullRequestReviewers.ROLE_OBSERVER, user)
 
     def notify_users(self, pull_request, updating_user, ancestor_commit_id,
                      commit_changes, file_changes):
 
         updating_user_id = updating_user.user_id
         reviewers = set([x.user.user_id for x in pull_request.get_pull_request_reviewers()])
         # NOTE(marcink): send notification to all other users except to
         # person who updated the PR
         recipients = reviewers.difference(set([updating_user_id]))
 
         log.debug('Notify following recipients about pull-request update %s', recipients)
 
         pull_request_obj = pull_request
 
         # send email about the update
         changed_files = (
                 file_changes.added + file_changes.modified + file_changes.removed)
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
                              repo_name=pr_target_repo.repo_name,
                              pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'updating_user': updating_user,
 
             'pull_request': pull_request_obj,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
 
             'ancestor_commit_id': ancestor_commit_id,
             'added_commits': commit_changes.added,
             'removed_commits': commit_changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
             'thread_ids': [pr_url],
         }
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=updating_user,
             notification_subject='',  # Filled in based on the notification_type
             notification_body='',  # Filled in based on the notification_type
             notification_type=EmailNotificationModel.TYPE_PULL_REQUEST_UPDATE,
             recipients=recipients,
             email_kwargs=email_kwargs,
         )
 
     def delete(self, pull_request, user=None):
         if not user:
             user = getattr(get_current_rhodecode_user(), 'username', None)
 
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         self._cleanup_merge_workspace(pull_request)
         self._log_audit_action(
             'repo.pull_request.delete', {'old_data': old_data},
             user, pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         self._cleanup_merge_workspace(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         self.trigger_pull_request_hook(pull_request, pull_request.author, 'close')
 
         pr_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
             self, pull_request, user, repo, message=None, auth_user=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
         if pull_request_review_status == ChangesetStatus.STATUS_APPROVED:
             # approved only if we have voting consent
             status = ChangesetStatus.STATUS_APPROVED
         else:
             status = ChangesetStatus.STATUS_REJECTED
         status_lbl = ChangesetStatus.get_status_lbl(status)
 
         default_message = (
             'Closing with status change {transition_icon} {status}.'
         ).format(transition_icon='>', status=status_lbl)
         text = message or default_message
 
         # create a comment, and link it to new status
         comment = CommentsModel().create(
             text=text,
             repo=repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
             closing_pr=True,
             auth_user=auth_user,
         )
 
         # calculate old status before we change it
         old_calculated_status = pull_request.calculated_review_status()
         ChangesetStatusModel().set_status(
             repo.repo_id,
             status,
             user.user_id,
             comment=comment,
             pull_request=pull_request.pull_request_id
         )
 
         Session().flush()
 
         self.trigger_pull_request_hook(pull_request, user, 'comment',
                                        data={'comment': comment})
 
         # we now calculate the status of pull request again, and based on that
         # calculation trigger status change. This might happen in cases
         # that non-reviewer admin closes a pr, which means his vote doesn't
         # change the status, while if he's a reviewer this might change it.
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             self.trigger_pull_request_hook(pull_request, user, 'review_status_change',
                                            data={'status': calculated_status})
 
         # finally close the PR
         PullRequestModel().close_pull_request(pull_request.pull_request_id, user)
 
         return comment, status
 
     def merge_status(self, pull_request, translator=None, force_shadow_repo_refresh=False):
         _ = translator or get_current_request().translate
 
         if not self._is_merge_enabled(pull_request):
             return None, False, _('Server-side pull request merging is disabled.')
 
         if pull_request.is_closed():
             return None, False, _('This pull request is closed.')
 
         merge_possible, msg = self._check_repo_requirements(
             target=pull_request.target_repo, source=pull_request.source_repo,
             translator=_)
         if not merge_possible:
             return None, merge_possible, msg
 
         try:
             merge_response = self._try_merge(
                 pull_request, force_shadow_repo_refresh=force_shadow_repo_refresh)
             log.debug("Merge response: %s", merge_response)
             return merge_response, merge_response.possible, merge_response.merge_status_message
         except NotImplementedError:
             return None, False, _('Pull request merging is not supported.')
 
     def _check_repo_requirements(self, target, source, translator):
         """
         Check if `target` and `source` have compatible requirements.
 
         Currently this is just checking for largefiles.
         """
         _ = translator
         target_has_largefiles = self._has_largefiles(target)
         source_has_largefiles = self._has_largefiles(source)
         merge_possible = True
         message = u''
 
         if target_has_largefiles != source_has_largefiles:
             merge_possible = False
             if source_has_largefiles:
                 message = _(
                     'Target repository large files support is disabled.')
             else:
                 message = _(
                     'Source repository large files support is disabled.')
 
         return merge_possible, message
 
     def _has_largefiles(self, repo):
         largefiles_ui = VcsSettingsModel(repo=repo).get_ui_settings(
             'extensions', 'largefiles')
         return largefiles_ui and largefiles_ui[0].active
 
     def _try_merge(self, pull_request, force_shadow_repo_refresh=False):
         """
         Try to merge the pull request and return the merge status.
         """
         log.debug(
             "Trying out if the pull request %s can be merged. Force_refresh=%s",
             pull_request.pull_request_id, force_shadow_repo_refresh)
         target_vcs = pull_request.target_repo.scm_instance()
         # Refresh the target reference.
         try:
             target_ref = self._refresh_reference(
                 pull_request.target_ref_parts, target_vcs)
         except CommitDoesNotExistError:
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF,
                 metadata={'target_ref': pull_request.target_ref_parts})
             return merge_state
 
         target_locked = pull_request.target_repo.locked
         if target_locked and target_locked[0]:
             locked_by = 'user:{}'.format(target_locked[0])
             log.debug("The target repository is locked by %s.", locked_by)
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_LOCKED,
                 metadata={'locked_by': locked_by})
         elif force_shadow_repo_refresh or self._needs_merge_state_refresh(
                 pull_request, target_ref):
             log.debug("Refreshing the merge status of the repository.")
             merge_state = self._refresh_merge_state(
                 pull_request, target_vcs, target_ref)
         else:
             possible = pull_request.last_merge_status == MergeFailureReason.NONE
             metadata = {
                 'unresolved_files': '',
                 'target_ref': pull_request.target_ref_parts,
                 'source_ref': pull_request.source_ref_parts,
             }
             if pull_request.last_merge_metadata:
                 metadata.update(pull_request.last_merge_metadata_parsed)
 
             if not possible and target_ref.type == 'branch':
                 # NOTE(marcink): case for mercurial multiple heads on branch
                 heads = target_vcs._heads(target_ref.name)
                 if len(heads) != 1:
                     heads = '\n,'.join(target_vcs._heads(target_ref.name))
                     metadata.update({
                         'heads': heads
                     })
 
             merge_state = MergeResponse(
                 possible, False, None, pull_request.last_merge_status, metadata=metadata)
 
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
         if reference.type in self.UPDATABLE_REF_TYPES:
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id
 
         refreshed_commit = vcs_repository.get_commit(name_or_id)
         refreshed_reference = Reference(
             reference.type, reference.name, refreshed_commit.raw_id)
         return refreshed_reference
 
     def _needs_merge_state_refresh(self, pull_request, target_reference):
         return not(
             pull_request.revisions and
             pull_request.revisions[0] == pull_request._last_merge_source_rev and
             target_reference.commit_id == pull_request._last_merge_target_rev)
 
     def _refresh_merge_state(self, pull_request, target_vcs, target_reference):
         workspace_id = self._workspace_id(pull_request)
         source_vcs = pull_request.source_repo.scm_instance()
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         merge_state = target_vcs.merge(
             repo_id, workspace_id,
             target_reference, source_vcs, pull_request.source_ref_parts,
             dry_run=True, use_rebase=use_rebase,
             close_branch=close_branch)
 
         # Do not store the response if there was an unknown error.
         if merge_state.failure_reason != MergeFailureReason.UNKNOWN:
             pull_request._last_merge_source_rev = \
                 pull_request.source_ref_parts.commit_id
             pull_request._last_merge_target_rev = target_reference.commit_id
             pull_request.last_merge_status = merge_state.failure_reason
             pull_request.last_merge_metadata = merge_state.metadata
 
             pull_request.shadow_merge_ref = merge_state.merge_ref
             Session().add(pull_request)
             Session().commit()
 
         return merge_state
 
     def _workspace_id(self, pull_request):
         workspace_id = 'pr-%s' % pull_request.pull_request_id
         return workspace_id
 
     def generate_repo_data(self, repo, commit_id=None, branch=None,
                            bookmark=None, translator=None):
         from rhodecode.model.repo import RepoModel
 
         all_refs, selected_ref = \
             self._get_repo_pullrequest_sources(
                 repo.scm_instance(), commit_id=commit_id,
                 branch=branch, bookmark=bookmark, translator=translator)
 
         refs_select2 = []
         for element in all_refs:
             children = [{'id': x[0], 'text': x[1]} for x in element[0]]
             refs_select2.append({'text': element[1], 'children': children})
 
         return {
             'user': {
                 'user_id': repo.user.user_id,
                 'username': repo.user.username,
                 'firstname': repo.user.first_name,
                 'lastname': repo.user.last_name,
                 'gravatar_link': h.gravatar_url(repo.user.email, 14),
             },
             'name': repo.repo_name,
             'link': RepoModel().get_url(repo),
             'description': h.chop_at_smart(repo.description_safe, '\n'),
             'refs': {
                 'all_refs': all_refs,
                 'selected_ref': selected_ref,
                 'select2_refs': refs_select2
             }
         }
 
     def generate_pullrequest_title(self, source, source_ref, target):
         return u'{source}#{at_ref} to {target}'.format(
             source=source,
             at_ref=source_ref,
             target=target,
         )
 
     def _cleanup_merge_workspace(self, pull_request):
         # Merging related cleanup
         repo_id = pull_request.target_repo.repo_id
         target_scm = pull_request.target_repo.scm_instance()
         workspace_id = self._workspace_id(pull_request)
 
         try:
             target_scm.cleanup_merge_workspace(repo_id, workspace_id)
         except NotImplementedError:
             pass
 
     def _get_repo_pullrequest_sources(
             self, repo, commit_id=None, branch=None, bookmark=None,
             translator=None):
         """
         Return a structure with repo's interesting commits, suitable for
         the selectors in pullrequest controller
 
         :param commit_id: a commit that must be in the list somehow
             and selected by default
         :param branch: a branch that must be in the list and selected
             by default - even if closed
         :param bookmark: a bookmark that must be in the list and selected
         """
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
         branch = safe_str(branch) if branch else None
         bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
         # order matters: first source that has commit_id in it will be selected
         sources = []
         sources.append(('book', repo.bookmarks.items(), _('Bookmarks'), bookmark))
         sources.append(('branch', repo.branches.items(), _('Branches'), branch))
 
         if commit_id:
             ref_commit = (h.short_id(commit_id), commit_id)
             sources.append(('rev', [ref_commit], _('Commit IDs'), commit_id))
 
         sources.append(
             ('branch', repo.branches_closed.items(), _('Closed Branches'), branch),
         )
 
         groups = []
 
         for group_key, ref_list, group_name, match in sources:
             group_refs = []
             for ref_name, ref_id in ref_list:
                 ref_key = u'{}:{}:{}'.format(group_key, ref_name, ref_id)
                 group_refs.append((ref_key, ref_name))
 
                 if not selected:
                     if set([commit_id, match]) & set([ref_id, ref_name]):
                         selected = ref_key
 
             if group_refs:
                 groups.append((group_refs, group_name))
 
         if not selected:
             ref = commit_id or branch or bookmark
             if ref:
                 raise CommitDoesNotExistError(
                     u'No commit refs could be found matching: {}'.format(ref))
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = u'branch:{}:{}'.format(
                     safe_str(repo.DEFAULT_BRANCH_NAME),
                     safe_str(repo.branches[repo.DEFAULT_BRANCH_NAME])
                 )
             elif repo.commit_ids:
                 # make the user select in this case
                 selected = None
             else:
                 raise EmptyRepositoryError()
         return groups, selected
 
     def get_diff(self, source_repo, source_ref_id, target_ref_id,
                  hide_whitespace_changes, diff_context):
 
         return self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
     def _get_diff_from_pr_or_version(
             self, source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context):
 
         target_commit = source_repo.get_commit(
             commit_id=safe_str(target_ref_id))
         source_commit = source_repo.get_commit(
             commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         if isinstance(source_repo, Repository):
             vcs_repo = source_repo.scm_instance()
         else:
             vcs_repo = source_repo
 
         # TODO: johbo: In the context of an update, we cannot reach
         # the old commit anymore with our normal mechanisms. It needs
         # some sort of special support in the vcs layer to avoid this
         # workaround.
         if (source_commit.raw_id == vcs_repo.EMPTY_COMMIT_ID and
                 vcs_repo.alias == 'git'):
             source_commit.raw_id = safe_str(source_ref_id)
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
                   safe_str(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit,
             ignore_whitespace=hide_whitespace_changes, context=diff_context)
         return vcs_diff
 
     def _is_merge_enabled(self, pull_request):
         return self._get_general_setting(
             pull_request, 'rhodecode_pr_merge_enabled')
 
     def _use_rebase_for_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_use_rebase_for_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_use_rebase_for_merging')
 
         return False
 
     def _user_name_for_merging(self, pull_request, user):
         env_user_name_attr = os.environ.get('RC_MERGE_USER_NAME_ATTR', '')
         if env_user_name_attr and hasattr(user, env_user_name_attr):
             user_name_attr = env_user_name_attr
         else:
             user_name_attr = 'short_contact'
 
         user_name = getattr(user, user_name_attr)
         return user_name
 
     def _close_branch_before_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_close_branch_before_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_close_branch_before_merging')
 
         return False
 
     def _get_general_setting(self, pull_request, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=pull_request.target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _log_audit_action(self, action, action_data, user, pull_request):
         audit_logger.store(
             action=action,
             action_data=action_data,
             user=user,
             repo=pull_request.target_repo)
 
     def get_reviewer_functions(self):
         """
         Fetches functions for validation and fetching default reviewers.
         If available we use the EE package, else we fallback to CE
         package functions
         """
         try:
             from rc_reviewers.utils import get_default_reviewers_data
             from rc_reviewers.utils import validate_default_reviewers
             from rc_reviewers.utils import validate_observers
         except ImportError:
             from rhodecode.apps.repository.utils import get_default_reviewers_data
             from rhodecode.apps.repository.utils import validate_default_reviewers
             from rhodecode.apps.repository.utils import validate_observers
 
         return get_default_reviewers_data, validate_default_reviewers, validate_observers
 
 
 class MergeCheck(object):
     """
     Perform Merge Checks and returns a check object which stores information
     about merge errors, and merge conditions
     """
     TODO_CHECK = 'todo'
     PERM_CHECK = 'perm'
     REVIEW_CHECK = 'review'
     MERGE_CHECK = 'merge'
     WIP_CHECK = 'wip'
 
     def __init__(self):
         self.review_status = None
         self.merge_possible = None
         self.merge_msg = ''
         self.merge_response = None
         self.failed = None
         self.errors = []
         self.error_details = OrderedDict()
         self.source_commit = AttributeDict()
         self.target_commit = AttributeDict()
         self.reviewers_count = 0
         self.observers_count = 0
 
     def __repr__(self):
         return '<MergeCheck(possible:{}, failed:{}, errors:{})>'.format(
             self.merge_possible, self.failed, self.errors)
 
     def push_error(self, error_type, message, error_key, details):
         self.failed = True
         self.errors.append([error_type, message])
         self.error_details[error_key] = dict(
             details=details,
             error_type=error_type,
             message=message
         )
 
     @classmethod
     def validate(cls, pull_request, auth_user, translator, fail_early=False,
                  force_shadow_repo_refresh=False):
         _ = translator
         merge_check = cls()
 
         # title has WIP:
         if pull_request.work_in_progress:
             log.debug("MergeCheck: cannot merge, title has wip: marker.")
 
             msg = _('WIP marker in title prevents from accidental merge.')
             merge_check.push_error('error', msg, cls.WIP_CHECK, pull_request.title)
             if fail_early:
                 return merge_check
 
         # permissions to merge
         user_allowed_to_merge = PullRequestModel().check_user_merge(pull_request, auth_user)
         if not user_allowed_to_merge:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('User `{}` not allowed to perform merge.').format(auth_user.username)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # permission to merge into the target branch
         target_commit_id = pull_request.target_ref_parts.commit_id
         if pull_request.target_ref_parts.type == 'branch':
             branch_name = pull_request.target_ref_parts.name
         else:
             # for mercurial we can always figure out the branch from the commit
             # in case of bookmark
             target_commit = pull_request.target_repo.get_commit(target_commit_id)
             branch_name = target_commit.branch
 
         rule, branch_perm = auth_user.get_rule_and_branch_permission(
             pull_request.target_repo.repo_name, branch_name)
         if branch_perm and branch_perm == 'branch.none':
             msg = _('Target branch `{}` changes rejected by rule {}.').format(
                 branch_name, rule)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # review status, must be always present
         review_status = pull_request.calculated_review_status()
         merge_check.review_status = review_status
         merge_check.reviewers_count = pull_request.reviewers_count
         merge_check.observers_count = pull_request.observers_count
 
         status_approved = review_status == ChangesetStatus.STATUS_APPROVED
         if not status_approved and merge_check.reviewers_count:
             log.debug("MergeCheck: cannot merge, approval is pending.")
             msg = _('Pull request reviewer approval is pending.')
 
             merge_check.push_error('warning', msg, cls.REVIEW_CHECK, review_status)
 
             if fail_early:
                 return merge_check
 
         # left over TODOs
         todos = CommentsModel().get_pull_request_unresolved_todos(pull_request)
         if todos:
             log.debug("MergeCheck: cannot merge, {} "
                       "unresolved TODOs left.".format(len(todos)))
 
             if len(todos) == 1:
                 msg = _('Cannot merge, {} TODO still not resolved.').format(
                     len(todos))
             else:
                 msg = _('Cannot merge, {} TODOs still not resolved.').format(
                     len(todos))
 
             merge_check.push_error('warning', msg, cls.TODO_CHECK, todos)
 
             if fail_early:
                 return merge_check
 
         # merge possible, here is the filesystem simulation + shadow repo
         merge_response, merge_status, msg = PullRequestModel().merge_status(
             pull_request, translator=translator,
             force_shadow_repo_refresh=force_shadow_repo_refresh)
 
         merge_check.merge_possible = merge_status
         merge_check.merge_msg = msg
         merge_check.merge_response = merge_response
 
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         try:
             source_commit, target_commit = PullRequestModel().get_flow_commits(pull_request)
             merge_check.source_commit.changed = source_ref_id != source_commit.raw_id
             merge_check.source_commit.ref_spec = pull_request.source_ref_parts
             merge_check.source_commit.current_raw_id = source_commit.raw_id
             merge_check.source_commit.previous_raw_id = source_ref_id
 
             merge_check.target_commit.changed = target_ref_id != target_commit.raw_id
             merge_check.target_commit.ref_spec = pull_request.target_ref_parts
             merge_check.target_commit.current_raw_id = target_commit.raw_id
             merge_check.target_commit.previous_raw_id = target_ref_id
         except (SourceRefMissing, TargetRefMissing):
             pass
 
         if not merge_status:
             log.debug("MergeCheck: cannot merge, pull request merge not possible.")
             merge_check.push_error('warning', msg, cls.MERGE_CHECK, None)
 
             if fail_early:
                 return merge_check
 
         log.debug('MergeCheck: is failed: %s', merge_check.failed)
         return merge_check
 
     @classmethod
     def get_merge_conditions(cls, pull_request, translator):
         _ = translator
         merge_details = {}
 
         model = PullRequestModel()
         use_rebase = model._use_rebase_for_merging(pull_request)
 
         if use_rebase:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: rebase')
             )
         else:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: explicit merge commit')
             )
 
         close_branch = model._close_branch_before_merging(pull_request)
         if close_branch:
             repo_type = pull_request.target_repo.repo_type
             close_msg = ''
             if repo_type == 'hg':
                 close_msg = _('Source branch will be closed before the merge.')
             elif repo_type == 'git':
                 close_msg = _('Source branch will be deleted after the merge.')
 
             merge_details['close_branch'] = dict(
                 details={},
                 message=close_msg
             )
 
         return merge_details
 
 
 @dataclasses.dataclass
 class ChangeTuple:
     added: list
     common: list
     removed: list
     total: list
 
 
 @dataclasses.dataclass
 class FileChangeTuple:
     added: list
     modified: list
     removed: list

rhodecode/tests/vcs/test_branches.py

@@ -1,147 +1,147 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import datetime
 
 import pytest
 
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestBranches(BackendTestMixin):
 
     def test_empty_repository_has_no_branches(self, vcsbackend):
         empty_repo = vcsbackend.create_repo()
         assert empty_repo.branches == {}
 
     def test_branches_all(self, vcsbackend):
         branch_count = {
             'git': 1,
             'hg': 1,
             'svn': 0,
         }
         assert len(self.repo.branches_all) == branch_count[vcsbackend.alias]
 
     def test_closed_branches(self):
         assert len(self.repo.branches_closed) == 0
 
     def test_simple(self, local_dt_to_utc):
         tip = self.repo.get_commit()
         assert tip.message == 'Changes...'
         assert tip.date == local_dt_to_utc(datetime.datetime(2010, 1, 1, 21))
 
     @pytest.mark.backends("git", "hg")
     def test_new_branch(self):
         # This check must not be removed to ensure the 'branches' LazyProperty
         # gets hit *before* the new 'foobar' branch got created:
         assert 'foobar' not in self.repo.branches
         self.imc.add(
             FileNode(b'docs/index.txt', content=b'Documentation\n')
         )
         foobar_tip = self.imc.commit(
-            message=u'New branch: foobar',
-            author=u'joe <[e-mail]>',
+            message='New branch: foobar',
+            author='joe <[e-mail]>',
             branch='foobar',
         )
         assert 'foobar' in self.repo.branches
         assert foobar_tip.branch == 'foobar'
 
     @pytest.mark.backends("git", "hg")
     def test_new_head(self):
         tip = self.repo.get_commit()
         self.imc.add(
             FileNode(b'docs/index.txt',
             content=b'Documentation\n')
         )
         foobar_tip = self.imc.commit(
-            message=u'New branch: foobar',
-            author=u'joe <[e-mail]>',
+            message='New branch: foobar',
+            author='joe <[e-mail]>',
             branch='foobar',
             parents=[tip],
         )
         self.imc.change(FileNode(
             b'docs/index.txt',
             content=b'Documentation\nand more...\n'))
         newtip = self.imc.commit(
             message=u'At default branch',
             author=u'joe <[e-mail]>',
             branch=foobar_tip.branch,
             parents=[foobar_tip],
         )
 
         newest_tip = self.imc.commit(
             message=u'Merged with %s' % foobar_tip.raw_id,
             author=u'joe <[e-mail]>',
             branch=self.backend_class.DEFAULT_BRANCH_NAME,
             parents=[newtip, foobar_tip],
         )
 
         assert newest_tip.branch == \
             self.backend_class.DEFAULT_BRANCH_NAME
 
     @pytest.mark.backends("git", "hg")
     def test_branch_with_slash_in_name(self):
         self.imc.add(FileNode(b'extrafile', content=b'Some data\n'))
         self.imc.commit(
             u'Branch with a slash!', author=u'joe <[e-mail]>',
             branch='issue/123')
         assert 'issue/123' in self.repo.branches
 
     @pytest.mark.backends("git", "hg")
     def test_branch_with_slash_in_name_and_similar_without(self):
         self.imc.add(FileNode(b'extrafile', content=b'Some data\n'))
         self.imc.commit(
             u'Branch with a slash!', author=u'joe <[e-mail]>',
             branch='issue/123')
         self.imc.add(FileNode(b'extrafile II', content=b'Some data\n'))
         self.imc.commit(
             u'Branch without a slash...', author=u'joe <[e-mail]>',
             branch='123')
         assert 'issue/123' in self.repo.branches
         assert '123' in self.repo.branches
 
 
 class TestSvnBranches(object):
 
     def test_empty_repository_has_no_tags_and_branches(self, vcsbackend_svn):
         empty_repo = vcsbackend_svn.create_repo()
         assert empty_repo.branches == {}
         assert empty_repo.tags == {}
 
     def test_missing_structure_has_no_tags_and_branches(self, vcsbackend_svn):
         repo = vcsbackend_svn.create_repo(number_of_commits=1)
         assert repo.branches == {}
         assert repo.tags == {}
 
     def test_discovers_ordered_branches(self, vcsbackend_svn):
         repo = vcsbackend_svn['svn-simple-layout']
         expected_branches = [
             'branches/add-docs',
             'branches/argparse',
             'trunk',
         ]
         assert list(repo.branches.keys()) == expected_branches
 
     def test_discovers_ordered_tags(self, vcsbackend_svn):
         repo = vcsbackend_svn['svn-simple-layout']
         expected_tags = [
             'tags/v0.1', 'tags/v0.2', 'tags/v0.3', 'tags/v0.5']
         assert list(repo.tags.keys()) == expected_tags