fix(file-caching): fixed cases when old cache was used before changes to operate on bytestrings

rhodecode/apps/_base/__init__.py

@@ -1,987 +1,985
 # Copyright (C) 2016-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 import operator
 
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest
 
 from rhodecode.lib import helpers as h, diffs, rc_cache
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib.utils2 import (
     StrictAttributeDict,
     str2bool,
     safe_int,
     datetime_to_time,
 )
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
 from rhodecode.model import repo_group
 from rhodecode.model import user_group
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel, IssueTrackerSettingsModel
 from rhodecode.model.repo import ReadmeFinder
 
 log = logging.getLogger(__name__)
 
 
 ADMIN_PREFIX: str = "/_admin"
 STATIC_FILE_PREFIX: str = "/_static"
 
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     "group_name": r".*?[^/]",
     "repo_group_name": r".*?[^/]",
     # repo names can have a slash in them, but they must not end with a slash
     "repo_name": r".*?[^/]",
     # file path eats up everything at the end
     "f_path": r".*",
     # reference types
     "source_ref_type": r"(branch|book|tag|rev|\%\(source_ref_type\)s)",
     "target_ref_type": r"(branch|book|tag|rev|\%\(target_ref_type\)s)",
 }
 
 
 def add_route_with_slash(config, name, pattern, **kw):
     config.add_route(name, pattern, **kw)
     if not pattern.endswith("/"):
         config.add_route(name + "_slash", pattern + "/", **kw)
 
 
 def add_route_requirements(route_path, requirements=None):
     """
     Adds regex requirements to pyramid routes using a mapping dict
     e.g::
         add_route_requirements('{repo_name}/settings')
     """
     requirements = requirements or URL_NAME_REQUIREMENTS
     for key, regex in list(requirements.items()):
         route_path = route_path.replace("{%s}" % key, "{%s:%s}" % (key, regex))
     return route_path
 
 
 def get_format_ref_id(repo):
     """Returns a `repo` specific reference formatter function"""
     if h.is_svn(repo):
         return _format_ref_id_svn
     else:
         return _format_ref_id
 
 
 def _format_ref_id(name, raw_id):
     """Default formatting of a given reference `name`"""
     return name
 
 
 def _format_ref_id_svn(name, raw_id):
     """Special way of formatting a reference for Subversion including path"""
     return f"{name}@{raw_id}"
 
 
 class TemplateArgs(StrictAttributeDict):
     pass
 
 
 class BaseAppView(object):
     DONT_CHECKOUT_VIEWS = ["channelstream_connect", "ops_ping"]
     EXTRA_VIEWS_TO_IGNORE = ['login', 'register', 'logout']
     SETUP_2FA_VIEW = 'setup_2fa'
     VERIFY_2FA_VIEW = 'check_2fa'
 
     def __init__(self, context, request):
         self.request = request
         self.context = context
         self.session = request.session
         if not hasattr(request, "user"):
             # NOTE(marcink): edge case, we ended up in matched route
             # but probably of web-app context, e.g API CALL/VCS CALL
             if hasattr(request, "vcs_call") or hasattr(request, "rpc_method"):
                 log.warning("Unable to process request `%s` in this scope", request)
                 raise HTTPBadRequest()
 
         self._rhodecode_user = request.user  # auth user
         self._rhodecode_db_user = self._rhodecode_user.get_instance()
         self.user_data = self._rhodecode_db_user.user_data if self._rhodecode_db_user else {}
         self._maybe_needs_password_change(
             request.matched_route.name, self._rhodecode_db_user
         )
         self._maybe_needs_2fa_configuration(
             request.matched_route.name, self._rhodecode_db_user
         )
         self._maybe_needs_2fa_check(
             request.matched_route.name, self._rhodecode_db_user
         )
 
     def _maybe_needs_password_change(self, view_name, user_obj):
         if view_name in self.DONT_CHECKOUT_VIEWS:
             return
 
         log.debug(
             "Checking if user %s needs password change on view %s", user_obj, view_name
         )
 
         skip_user_views = [
             "logout",
             "login",
             "check_2fa",
             "my_account_password",
             "my_account_password_update",
         ]
 
         if not user_obj:
             return
 
         if user_obj.username == User.DEFAULT_USER:
             return
 
         now = time.time()
         should_change = self.user_data.get("force_password_change")
         change_after = safe_int(should_change) or 0
         if should_change and now > change_after:
             log.debug("User %s requires password change", user_obj)
             h.flash(
                 "You are required to change your password",
                 "warning",
                 ignore_duplicate=True,
             )
 
             if view_name not in skip_user_views:
                 raise HTTPFound(self.request.route_path("my_account_password"))
 
     def _maybe_needs_2fa_configuration(self, view_name, user_obj):
         if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
             return
 
         if not user_obj:
             return
 
         if user_obj.needs_2fa_configure and view_name != self.SETUP_2FA_VIEW:
             h.flash(
                 "You are required to configure 2FA",
                 "warning",
                 ignore_duplicate=False,
             )
             # Special case for users created "on the fly" (ldap case for new user)
             user_obj.check_2fa_required = False
             raise HTTPFound(self.request.route_path(self.SETUP_2FA_VIEW))
 
     def _maybe_needs_2fa_check(self, view_name, user_obj):
         if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
             return
 
         if not user_obj:
             return
 
         if user_obj.check_2fa_required and view_name != self.VERIFY_2FA_VIEW:
             raise HTTPFound(self.request.route_path(self.VERIFY_2FA_VIEW))
 
     def _log_creation_exception(self, e, repo_name):
         _ = self.request.translate
         reason = None
         if len(e.args) == 2:
             reason = e.args[1]
 
         if reason == "INVALID_CERTIFICATE":
             log.exception("Exception creating a repository: invalid certificate")
             msg = _("Error creating repository %s: invalid certificate") % repo_name
         else:
             log.exception("Exception creating a repository")
             msg = _("Error creating repository %s") % repo_name
         return msg
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         c = TemplateArgs()
         c.auth_user = self.request.user
         # TODO(marcink): migrate the usage of c.rhodecode_user to c.auth_user
         c.rhodecode_user = self.request.user
 
         if include_app_defaults:
             from rhodecode.lib.base import attach_context_attributes
 
             attach_context_attributes(c, self.request, self.request.user.user_id)
 
         c.is_super_admin = c.auth_user.is_admin
 
         c.can_create_repo = c.is_super_admin
         c.can_create_repo_group = c.is_super_admin
         c.can_create_user_group = c.is_super_admin
 
         c.is_delegated_admin = False
 
         if not c.auth_user.is_default and not c.is_super_admin:
             c.can_create_repo = h.HasPermissionAny("hg.create.repository")(
                 user=self.request.user
             )
             repositories = c.auth_user.repositories_admin or c.can_create_repo
 
             c.can_create_repo_group = h.HasPermissionAny("hg.repogroup.create.true")(
                 user=self.request.user
             )
             repository_groups = (
                 c.auth_user.repository_groups_admin or c.can_create_repo_group
             )
 
             c.can_create_user_group = h.HasPermissionAny("hg.usergroup.create.true")(
                 user=self.request.user
             )
             user_groups = c.auth_user.user_groups_admin or c.can_create_user_group
             # delegated admin can create, or manage some objects
             c.is_delegated_admin = repositories or repository_groups or user_groups
         return c
 
     def _get_template_context(self, tmpl_args, **kwargs):
         local_tmpl_args = {"defaults": {}, "errors": {}, "c": tmpl_args}
         local_tmpl_args.update(kwargs)
         return local_tmpl_args
 
     def load_default_context(self):
         """
         example:
 
         def load_default_context(self):
             c = self._get_local_tmpl_context()
             c.custom_var = 'foobar'
 
             return c
         """
         raise NotImplementedError("Needs implementation in view class")
 
 
 class RepoAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_repo = request.db_repo
         self.db_repo_name = self.db_repo.repo_name
         self.db_repo_pull_requests = ScmModel().get_pull_requests(self.db_repo)
         self.db_repo_artifacts = ScmModel().get_artifacts(self.db_repo)
         self.db_repo_patterns = IssueTrackerSettingsModel(repo=self.db_repo)
 
     def _handle_missing_requirements(self, error):
         log.error(
             "Requirements are missing for repository %s: %s",
             self.db_repo_name,
             safe_str(error),
         )
 
     def _prepare_and_set_clone_url(self, c):
         username = ""
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = self._rhodecode_user.username
 
         _def_clone_uri = c.clone_uri_tmpl
         _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri
         )
         c.clone_repo_url_id = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri_id
         )
         c.clone_repo_url_ssh = self.db_repo.clone_url(
             uri_tmpl=_def_clone_uri_ssh, ssh=True
         )
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super()._get_local_tmpl_context(include_app_defaults=include_app_defaults)
 
         # register common vars for this type of view
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
         c.repository_artifacts = self.db_repo_artifacts
         c.repository_is_user_following = ScmModel().is_following_repo(
             self.db_repo_name, self._rhodecode_user.user_id
         )
         self.path_filter = PathFilter(None)
 
         c.repository_requirements_missing = {}
         try:
             self.rhodecode_vcs_repo = self.db_repo.scm_instance()
             # NOTE(marcink):
             # comparison to None since if it's an object __bool__ is expensive to
             # calculate
             if self.rhodecode_vcs_repo is not None:
                 path_perms = self.rhodecode_vcs_repo.get_path_permissions(
                     c.auth_user.username
                 )
                 self.path_filter = PathFilter(path_perms)
         except RepositoryRequirementError as e:
             c.repository_requirements_missing = {"error": str(e)}
             self._handle_missing_requirements(e)
             self.rhodecode_vcs_repo = None
 
         c.path_filter = self.path_filter  # used by atom_feed_entry.mako
 
         if self.rhodecode_vcs_repo is None:
             # unable to fetch this repo as vcs instance, report back to user
             log.debug(
                 "Repository was not found on filesystem, check if it exists or is not damaged"
             )
             h.flash(
                 _(
                     "The repository `%(repo_name)s` cannot be loaded in filesystem. "
                     "Please check if it exist, or is not damaged."
                 )
                 % {"repo_name": c.repo_name},
                 category="error",
                 ignore_duplicate=True,
             )
             if c.repository_requirements_missing:
                 route = self.request.matched_route.name
                 if route.startswith(("edit_repo", "repo_summary")):
                     # allow summary and edit repo on missing requirements
                     return c
 
                 raise HTTPFound(
                     h.route_path("repo_summary", repo_name=self.db_repo_name)
                 )
 
             else:  # redirect if we don't show missing requirements
                 raise HTTPFound(h.route_path("home"))
 
         c.has_origin_repo_read_perm = False
         if self.db_repo.fork:
             c.has_origin_repo_read_perm = h.HasRepoPermissionAny(
                 "repository.write", "repository.read", "repository.admin"
             )(self.db_repo.fork.repo_name, "summary fork link")
 
         return c
 
     def _get_f_path_unchecked(self, matchdict, default=None):
         """
         Should only be used by redirects, everything else should call _get_f_path
         """
         f_path = matchdict.get("f_path")
         if f_path:
             # fix for multiple initial slashes that causes errors for GIT
             return f_path.lstrip("/")
 
         return default
 
     def _get_f_path(self, matchdict, default=None):
         f_path_match = self._get_f_path_unchecked(matchdict, default)
         return self.path_filter.assert_path_permissions(f_path_match)
 
     def _get_general_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _get_repo_setting(self, target_repo, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=target_repo)
         settings = settings_model.get_repo_settings_inherited()
         return settings.get(settings_key, default)
 
-    def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path="/"):
+    def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path="/", nodes=None):
         log.debug("Looking for README file at path %s", path)
         if commit_id:
             landing_commit_id = commit_id
         else:
             landing_commit = db_repo.get_landing_commit()
             if isinstance(landing_commit, EmptyCommit):
                 return None, None
             landing_commit_id = landing_commit.raw_id
 
         cache_namespace_uid = f"repo.{db_repo.repo_id}"
         region = rc_cache.get_or_create_region(
             "cache_repo", cache_namespace_uid, use_async_runner=False
         )
         start = time.time()
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
         def generate_repo_readme(
-            repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type
+            _repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type
         ):
-            readme_data = None
-            readme_filename = None
+            _readme_data = None
+            _readme_filename = None
 
             commit = db_repo.get_commit(_commit_id)
             log.debug("Searching for a README file at commit %s.", _commit_id)
-            readme_node = ReadmeFinder(_renderer_type).search(
-                commit, path=_readme_search_path
-            )
+            readme_node = ReadmeFinder(_renderer_type).search(commit, path=_readme_search_path, nodes=nodes)
 
             if readme_node:
                 log.debug("Found README node: %s", readme_node)
 
                 relative_urls = {
                     "raw": h.route_path(
                         "repo_file_raw",
                         repo_name=_repo_name,
                         commit_id=commit.raw_id,
                         f_path=readme_node.path,
                     ),
                     "standard": h.route_path(
                         "repo_files",
                         repo_name=_repo_name,
                         commit_id=commit.raw_id,
                         f_path=readme_node.path,
                     ),
                 }
 
-                readme_data = self._render_readme_or_none(
+                _readme_data = self._render_readme_or_none(
                     commit, readme_node, relative_urls
                 )
-                readme_filename = readme_node.str_path
+                _readme_filename = readme_node.str_path
 
-            return readme_data, readme_filename
+            return _readme_data, _readme_filename
 
         readme_data, readme_filename = generate_repo_readme(
             db_repo.repo_id,
             landing_commit_id,
             db_repo.repo_name,
             path,
-            renderer_type,
+            renderer_type
         )
 
         compute_time = time.time() - start
         log.debug(
             "Repo README for path %s generated and computed in %.4fs",
             path,
             compute_time,
         )
         return readme_data, readme_filename
 
     def _render_readme_or_none(self, commit, readme_node, relative_urls):
         log.debug("Found README file `%s` rendering...", readme_node.path)
         renderer = MarkupRenderer()
         try:
             html_source = renderer.render(
                 readme_node.str_content, filename=readme_node.path
             )
             if relative_urls:
                 return relative_links(html_source, relative_urls)
             return html_source
         except Exception:
             log.exception("Exception while trying to render the README")
 
     def get_recache_flag(self):
         for flag_name in ["force_recache", "force-recache", "no-cache"]:
             flag_val = self.request.GET.get(flag_name)
             if str2bool(flag_val):
                 return True
         return False
 
     def get_commit_preload_attrs(cls):
         pre_load = [
             "author",
             "branch",
             "date",
             "message",
             "parents",
             "obsolete",
             "phase",
             "hidden",
         ]
         return pre_load
 
 
 class PathFilter(object):
     # Expects and instance of BasePathPermissionChecker or None
     def __init__(self, permission_checker):
         self.permission_checker = permission_checker
 
     def assert_path_permissions(self, path):
         if self.path_access_allowed(path):
             return path
         raise HTTPForbidden()
 
     def path_access_allowed(self, path):
         log.debug("Checking ACL permissions for PathFilter for `%s`", path)
         if self.permission_checker:
             has_access = path and self.permission_checker.has_access(path)
             log.debug(
                 "ACL Permissions checker enabled, ACL Check has_access: %s", has_access
             )
             return has_access
 
         log.debug("ACL permissions checker not enabled, skipping...")
         return True
 
     def filter_patchset(self, patchset):
         if not self.permission_checker or not patchset:
             return patchset, False
         had_filtered = False
         filtered_patchset = []
         for patch in patchset:
             filename = patch.get("filename", None)
             if not filename or self.permission_checker.has_access(filename):
                 filtered_patchset.append(patch)
             else:
                 had_filtered = True
         if had_filtered:
             if isinstance(patchset, diffs.LimitedDiffContainer):
                 filtered_patchset = diffs.LimitedDiffContainer(
                     patchset.diff_limit, patchset.cur_diff_size, filtered_patchset
                 )
             return filtered_patchset, True
         else:
             return patchset, False
 
     def render_patchset_filtered(
         self, diffset, patchset, source_ref=None, target_ref=None
     ):
         filtered_patchset, has_hidden_changes = self.filter_patchset(patchset)
         result = diffset.render_patchset(
             filtered_patchset, source_ref=source_ref, target_ref=target_ref
         )
         result.has_hidden_changes = has_hidden_changes
         return result
 
     def get_raw_patch(self, diff_processor):
         if self.permission_checker is None:
             return diff_processor.as_raw()
         elif self.permission_checker.has_full_access:
             return diff_processor.as_raw()
         else:
             return "# Repository has user-specific filters, raw patch generation is disabled."
 
     @property
     def is_enabled(self):
         return self.permission_checker is not None
 
 
 class RepoGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_repo_group = request.db_repo_group
         self.db_repo_group_name = self.db_repo_group.group_name
 
     def _get_local_tmpl_context(self, include_app_defaults=True):
         _ = self.request.translate
         c = super()._get_local_tmpl_context(include_app_defaults=include_app_defaults)
         c.repo_group = self.db_repo_group
         return c
 
     def _revoke_perms_on_yourself(self, form_result):
         _updates = [
             u
             for u in form_result["perm_updates"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         _additions = [
             u
             for u in form_result["perm_additions"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         _deletions = [
             u
             for u in form_result["perm_deletions"]
             if self._rhodecode_user.user_id == int(u[0])
         ]
         admin_perm = "group.admin"
         if (
             _updates
             and _updates[0][1] != admin_perm
             or _additions
             and _additions[0][1] != admin_perm
             or _deletions
             and _deletions[0][1] != admin_perm
         ):
             return True
         return False
 
 
 class UserGroupAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_user_group = request.db_user_group
         self.db_user_group_name = self.db_user_group.users_group_name
 
 
 class UserAppView(BaseAppView):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.db_user = request.db_user
         self.db_user_id = self.db_user.user_id
 
         _ = self.request.translate
         if not request.db_user_supports_default:
             if self.db_user.username == User.DEFAULT_USER:
                 h.flash(
                     _("Editing user `{}` is disabled.".format(User.DEFAULT_USER)),
                     category="warning",
                 )
                 raise HTTPFound(h.route_path("users"))
 
 
 class DataGridAppView(object):
     """
     Common class to have re-usable grid rendering components
     """
 
     def _extract_ordering(self, request, column_map=None):
         column_map = column_map or {}
         column_index = safe_int(request.GET.get("order[0][column]"))
         order_dir = request.GET.get("order[0][dir]", "desc")
         order_by = request.GET.get("columns[%s][data][sort]" % column_index, "name_raw")
 
         # translate datatable to DB columns
         order_by = column_map.get(order_by) or order_by
 
         search_q = request.GET.get("search[value]")
         return search_q, order_by, order_dir
 
     def _extract_chunk(self, request):
         start = safe_int(request.GET.get("start"), 0)
         length = safe_int(request.GET.get("length"), 25)
         draw = safe_int(request.GET.get("draw"))
         return draw, start, length
 
     def _get_order_col(self, order_by, model):
         if isinstance(order_by, str):
             try:
                 return operator.attrgetter(order_by)(model)
             except AttributeError:
                 return None
         else:
             return order_by
 
 
 class BaseReferencesView(RepoAppView):
     """
     Base for reference view for branches, tags and bookmarks.
     """
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     def load_refs_context(self, ref_items, partials_template):
         _render = self.request.get_partial_renderer(partials_template)
         pre_load = ["author", "date", "message", "parents"]
 
         is_svn = h.is_svn(self.rhodecode_vcs_repo)
         is_hg = h.is_hg(self.rhodecode_vcs_repo)
 
         format_ref_id = get_format_ref_id(self.rhodecode_vcs_repo)
 
         closed_refs = {}
         if is_hg:
             closed_refs = self.rhodecode_vcs_repo.branches_closed
 
         data = []
         for ref_name, commit_id in ref_items:
             commit = self.rhodecode_vcs_repo.get_commit(
                 commit_id=commit_id, pre_load=pre_load
             )
             closed = ref_name in closed_refs
 
             # TODO: johbo: Unify generation of reference links
             use_commit_id = "/" in ref_name or is_svn
 
             if use_commit_id:
                 files_url = h.route_path(
                     "repo_files",
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else "",
                     commit_id=commit_id,
                     _query=dict(at=ref_name),
                 )
 
             else:
                 files_url = h.route_path(
                     "repo_files",
                     repo_name=self.db_repo_name,
                     f_path=ref_name if is_svn else "",
                     commit_id=ref_name,
                     _query=dict(at=ref_name),
                 )
 
             data.append(
                 {
                     "name": _render("name", ref_name, files_url, closed),
                     "name_raw": ref_name,
                     "closed": closed,
                     "date": _render("date", commit.date),
                     "date_raw": datetime_to_time(commit.date),
                     "author": _render("author", commit.author),
                     "commit": _render(
                         "commit", commit.message, commit.raw_id, commit.idx
                     ),
                     "commit_raw": commit.idx,
                     "compare": _render(
                         "compare", format_ref_id(ref_name, commit.raw_id)
                     ),
                 }
             )
 
         return data
 
 
 class RepoRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         repo_name = info["match"]["repo_name"]
 
         repo_name_parts = repo_name.split("/")
         repo_slugs = [x for x in (repo_name_slug(x) for x in repo_name_parts)]
 
         if repo_name_parts != repo_slugs:
             # short-skip if the repo-name doesn't follow slug rule
             log.warning(
                 "repo_name: %s is different than slug %s", repo_name_parts, repo_slugs
             )
             return False
 
         repo_model = repo.RepoModel()
 
         by_name_match = repo_model.get_by_repo_name(repo_name, cache=False)
 
         def redirect_if_creating(route_info, db_repo):
             skip_views = ["edit_repo_advanced_delete"]
             route = route_info["route"]
             # we should skip delete view so we can actually "remove" repositories
             # if they get stuck in creating state.
             if route.name in skip_views:
                 return
 
             if db_repo.repo_state in [repo.Repository.STATE_PENDING]:
                 repo_creating_url = request.route_path(
                     "repo_creating", repo_name=db_repo.repo_name
                 )
                 raise HTTPFound(repo_creating_url)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo = by_name_match
             request.db_repo_name = request.db_repo.repo_name
 
             redirect_if_creating(info, by_name_match)
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             request.db_repo = by_id_match
             request.db_repo_name = request.db_repo.repo_name
             redirect_if_creating(info, by_id_match)
             return True
 
         return False
 
 
 class RepoForbidArchivedRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_forbid_archived = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         _ = request.translate
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             "%s checking if archived flag for repo for %s",
             self.__class__.__name__,
             rhodecode_db_repo.repo_name,
         )
 
         if rhodecode_db_repo.archived:
             log.warning(
                 "Current view is not supported for archived repo:%s",
                 rhodecode_db_repo.repo_name,
             )
 
             h.flash(
                 h.literal(_("Action not supported for archived repository.")),
                 category="warning",
             )
             summary_url = request.route_path(
                 "repo_summary", repo_name=rhodecode_db_repo.repo_name
             )
             raise HTTPFound(summary_url)
         return True
 
 
 class RepoTypeRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val or ["hg", "git", "svn"]
 
     def text(self):
         return f"repo_accepted_type = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         rhodecode_db_repo = request.db_repo
 
         log.debug(
             "%s checking repo type for %s in %s",
             self.__class__.__name__,
             rhodecode_db_repo.repo_type,
             self.val,
         )
 
         if rhodecode_db_repo.repo_type in self.val:
             return True
         else:
             log.warning(
                 "Current view is not supported for repo type:%s",
                 rhodecode_db_repo.repo_type,
             )
             return False
 
 
 class RepoGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"repo_group_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         repo_group_name = info["match"]["repo_group_name"]
 
         repo_group_name_parts = repo_group_name.split("/")
         repo_group_slugs = [
             x for x in [repo_name_slug(x) for x in repo_group_name_parts]
         ]
         if repo_group_name_parts != repo_group_slugs:
             # short-skip if the repo-name doesn't follow slug rule
             log.warning(
                 "repo_group_name: %s is different than slug %s",
                 repo_group_name_parts,
                 repo_group_slugs,
             )
             return False
 
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name, cache=False)
 
         if by_name_match:
             # register this as request object we can re-use later
             request.db_repo_group = by_name_match
             request.db_repo_group_name = request.db_repo_group.group_name
             return True
 
         return False
 
 
 class UserGroupRoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f"user_group_route = {self.val}"
 
     phash = text
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         user_group_id = info["match"]["user_group_id"]
         user_group_model = user_group.UserGroup()
         by_id_match = user_group_model.get(user_group_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user_group = by_id_match
             return True
 
         return False
 
 
 class UserRoutePredicateBase(object):
     supports_default = None
 
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         raise NotImplementedError()
 
     def __call__(self, info, request):
         if hasattr(request, "vcs_call"):
             # skip vcs calls
             return
 
         user_id = info["match"]["user_id"]
         user_model = user.User()
         by_id_match = user_model.get(user_id, cache=False)
 
         if by_id_match:
             # register this as request object we can re-use later
             request.db_user = by_id_match
             request.db_user_supports_default = self.supports_default
             return True
 
         return False
 
 
 class UserRoutePredicate(UserRoutePredicateBase):
     supports_default = False
 
     def text(self):
         return f"user_route = {self.val}"
 
     phash = text
 
 
 class UserRouteWithDefaultPredicate(UserRoutePredicateBase):
     supports_default = True
 
     def text(self):
         return f"user_with_default_route = {self.val}"
 
     phash = text
 
 
 def includeme(config):
     config.add_route_predicate("repo_route", RepoRoutePredicate)
     config.add_route_predicate("repo_accepted_types", RepoTypeRoutePredicate)
     config.add_route_predicate(
         "repo_forbid_when_archived", RepoForbidArchivedRoutePredicate
     )
     config.add_route_predicate("repo_group_route", RepoGroupRoutePredicate)
     config.add_route_predicate("user_group_route", UserGroupRoutePredicate)
     config.add_route_predicate("user_route_with_default", UserRouteWithDefaultPredicate)
     config.add_route_predicate("user_route", UserRoutePredicate)

rhodecode/apps/repository/views/repo_files.py

@@ -1,1704 +1,1601
 # Copyright (C) 2011-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import os
 import collections
 import urllib.request
 import urllib.parse
 import urllib.error
 import pathlib
 import time
 import random
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
 
 from pyramid.renderers import render
 from pyramid.response import Response
 
 import rhodecode
 from rhodecode.apps._base import RepoAppView
 
 
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.archive_cache import (
-    get_archival_cache_store, get_archival_config, ArchiveCacheGenerationLock, archive_iterator)
+    get_archival_cache_store,
+    get_archival_config,
+    ArchiveCacheGenerationLock,
+    archive_iterator,
+)
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
-from rhodecode.lib.codeblocks import (
-    filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
+from rhodecode.lib.codeblocks import filenode_as_lines_tokens, filenode_as_annotated_lines_tokens
 from rhodecode.lib.utils2 import convert_line_endings, detect_mode
 from rhodecode.lib.type_utils import str2bool
 from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
-from rhodecode.lib.auth import (
-    LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
+from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import (
-    RepositoryError, CommitDoesNotExistError, EmptyRepositoryError,
-    ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,
-    NodeDoesNotExistError, CommitError, NodeError)
+    RepositoryError,
+    CommitDoesNotExistError,
+    EmptyRepositoryError,
+    ImproperArchiveTypeError,
+    VCSError,
+    NodeAlreadyExistsError,
+    NodeDoesNotExistError,
+    CommitError,
+    NodeError,
+)
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Repository
 
 log = logging.getLogger(__name__)
 
 
-def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
+def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha="", with_hash=True):
     # original backward compat name of archive
-    clean_name = safe_str(convert_special_chars(db_repo_name).replace('/', '_'))
+    clean_name = safe_str(convert_special_chars(db_repo_name).replace("/", "_"))
 
     # e.g vcsserver-id-abcd-sub-1-abcfdef-archive-all.zip
     # vcsserver-id-abcd-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip
     id_sha = sha1_safe(str(db_repo_id))[:4]
-    sub_repo = 'sub-1' if subrepos else 'sub-0'
-    commit = commit_sha if with_hash else 'archive'
-    path_marker = (path_sha if with_hash else '') or 'all'
-    archive_name = f'{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}'
+    sub_repo = "sub-1" if subrepos else "sub-0"
+    commit = commit_sha if with_hash else "archive"
+    path_marker = (path_sha if with_hash else "") or "all"
+    archive_name = f"{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}"
 
     return archive_name
 
 
 def get_path_sha(at_path):
     return safe_str(sha1_safe(at_path)[:8])
 
 
 def _get_archive_spec(fname):
-    log.debug('Detecting archive spec for: `%s`', fname)
+    log.debug("Detecting archive spec for: `%s`", fname)
 
     fileformat = None
     ext = None
     content_type = None
     for a_type, content_type, extension in settings.ARCHIVE_SPECS:
-
         if fname.endswith(extension):
             fileformat = a_type
-            log.debug('archive is of type: %s', fileformat)
+            log.debug("archive is of type: %s", fileformat)
             ext = extension
             break
 
     if not fileformat:
         raise ValueError()
 
     # left over part of whole fname is the commit
-    commit_id = fname[:-len(ext)]
+    commit_id = fname[: -len(ext)]
 
     return commit_id, ext, fileformat, content_type
 
 
 class RepoFilesView(RepoAppView):
-
     @staticmethod
     def adjust_file_path_for_svn(f_path, repo):
         """
         Computes the relative path of `f_path`.
 
         This is mainly based on prefix matching of the recognized tags and
         branches in the underlying repository.
         """
-        tags_and_branches = itertools.chain(
-            repo.branches.keys(),
-            repo.tags.keys())
+        tags_and_branches = itertools.chain(repo.branches.keys(), repo.tags.keys())
         tags_and_branches = sorted(tags_and_branches, key=len, reverse=True)
 
         for name in tags_and_branches:
-            if f_path.startswith(f'{name}/'):
+            if f_path.startswith(f"{name}/"):
                 f_path = vcspath.relpath(f_path, name)
                 break
         return f_path
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
         c.enable_downloads = self.db_repo.enable_downloads
         return c
 
-    def _ensure_not_locked(self, commit_id='tip'):
+    def _ensure_not_locked(self, commit_id="tip"):
         _ = self.request.translate
 
         repo = self.db_repo
         if repo.enable_locking and repo.locked[0]:
-            h.flash(_('This repository has been locked by %s on %s')
-                    % (h.person_by_id(repo.locked[0]),
-                    h.format_date(h.time_to_datetime(repo.locked[1]))),
-                    'warning')
-            files_url = h.route_path(
-                'repo_files:default_path',
-                repo_name=self.db_repo_name, commit_id=commit_id)
+            h.flash(
+                _("This repository has been locked by %s on %s")
+                % (h.person_by_id(repo.locked[0]), h.format_date(h.time_to_datetime(repo.locked[1]))),
+                "warning",
+            )
+            files_url = h.route_path("repo_files:default_path", repo_name=self.db_repo_name, commit_id=commit_id)
             raise HTTPFound(files_url)
 
-    def forbid_non_head(self, is_head, f_path, commit_id='tip', json_mode=False):
+    def forbid_non_head(self, is_head, f_path, commit_id="tip", json_mode=False):
         _ = self.request.translate
 
         if not is_head:
-            message = _('Cannot modify file. '
-                        'Given commit `{}` is not head of a branch.').format(commit_id)
-            h.flash(message, category='warning')
+            message = _("Cannot modify file. " "Given commit `{}` is not head of a branch.").format(commit_id)
+            h.flash(message, category="warning")
 
             if json_mode:
                 return message
 
-            files_url = h.route_path(
-                'repo_files', repo_name=self.db_repo_name, commit_id=commit_id,
-                f_path=f_path)
+            files_url = h.route_path("repo_files", repo_name=self.db_repo_name, commit_id=commit_id, f_path=f_path)
             raise HTTPFound(files_url)
 
-    def check_branch_permission(self, branch_name, commit_id='tip', json_mode=False):
+    def check_branch_permission(self, branch_name, commit_id="tip", json_mode=False):
         _ = self.request.translate
 
-        rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission(
-            self.db_repo_name, branch_name)
-        if branch_perm and branch_perm not in ['branch.push', 'branch.push_force']:
-            message = _('Branch `{}` changes forbidden by rule {}.').format(
-                h.escape(branch_name), h.escape(rule))
-            h.flash(message, 'warning')
+        rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission(self.db_repo_name, branch_name)
+        if branch_perm and branch_perm not in ["branch.push", "branch.push_force"]:
+            message = _("Branch `{}` changes forbidden by rule {}.").format(h.escape(branch_name), h.escape(rule))
+            h.flash(message, "warning")
 
             if json_mode:
                 return message
 
-            files_url = h.route_path(
-                'repo_files:default_path', repo_name=self.db_repo_name, commit_id=commit_id)
+            files_url = h.route_path("repo_files:default_path", repo_name=self.db_repo_name, commit_id=commit_id)
 
             raise HTTPFound(files_url)
 
     def _get_commit_and_path(self):
         default_commit_id = self.db_repo.landing_ref_name
-        default_f_path = '/'
+        default_f_path = "/"
 
-        commit_id = self.request.matchdict.get('commit_id', default_commit_id)
+        commit_id = self.request.matchdict.get("commit_id", default_commit_id)
         f_path = self._get_f_path(self.request.matchdict, default_f_path)
 
         bytes_path = safe_bytes(f_path)
         return commit_id, f_path, bytes_path
 
     @classmethod
     def _get_default_encoding(cls, c):
-        enc_list = getattr(c, 'default_encodings', [])
-        return enc_list[0] if enc_list else 'UTF-8'
+        enc_list = getattr(c, "default_encodings", [])
+        return enc_list[0] if enc_list else "UTF-8"
 
     def _get_commit_or_redirect(self, commit_id, redirect_after=True):
         """
         This is a safe way to get commit. If an error occurs it redirects to
         tip with proper message
 
         :param commit_id: id of commit to fetch
         :param redirect_after: toggle redirection
         """
         _ = self.request.translate
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id)
         except EmptyRepositoryError:
             if not redirect_after:
                 return None
 
             add_new = upload_new = ""
-            if h.HasRepoPermissionAny(
-                    'repository.write', 'repository.admin')(self.db_repo_name):
-                _url = h.route_path(
-                    'repo_files_add_file',
-                    repo_name=self.db_repo_name, commit_id=0, f_path='')
-                add_new = h.link_to(
-                    _('add a new file'), _url, class_="alert-link")
+            if h.HasRepoPermissionAny("repository.write", "repository.admin")(self.db_repo_name):
+                _url = h.route_path("repo_files_add_file", repo_name=self.db_repo_name, commit_id=0, f_path="")
+                add_new = h.link_to(_("add a new file"), _url, class_="alert-link")
 
-                _url_upld = h.route_path(
-                    'repo_files_upload_file',
-                    repo_name=self.db_repo_name, commit_id=0, f_path='')
-                upload_new = h.link_to(
-                    _('upload a new file'), _url_upld, class_="alert-link")
+                _url_upld = h.route_path("repo_files_upload_file", repo_name=self.db_repo_name, commit_id=0, f_path="")
+                upload_new = h.link_to(_("upload a new file"), _url_upld, class_="alert-link")
 
-            h.flash(h.literal(
-                _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning')
-            raise HTTPFound(
-                h.route_path('repo_summary', repo_name=self.db_repo_name))
+            h.flash(
+                h.literal(_("There are no files yet. Click here to %s or %s.") % (add_new, upload_new)),
+                category="warning",
+            )
+            raise HTTPFound(h.route_path("repo_summary", repo_name=self.db_repo_name))
 
         except (CommitDoesNotExistError, LookupError) as e:
-            msg = _('No such commit exists for this repository. Commit: {}').format(commit_id)
-            h.flash(msg, category='error')
+            msg = _("No such commit exists for this repository. Commit: {}").format(commit_id)
+            h.flash(msg, category="error")
             raise HTTPNotFound()
         except RepositoryError as e:
-            h.flash(h.escape(safe_str(e)), category='error')
+            h.flash(h.escape(safe_str(e)), category="error")
             raise HTTPNotFound()
 
     def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
         """
         _ = self.request.translate
 
         try:
             file_node = commit_obj.get_node(path, pre_load=pre_load)
             if file_node.is_dir():
-                raise RepositoryError('The given path is a directory')
+                raise RepositoryError("The given path is a directory")
         except CommitDoesNotExistError:
-            log.exception('No such commit exists for this repository')
-            h.flash(_('No such commit exists for this repository'), category='error')
+            log.exception("No such commit exists for this repository")
+            h.flash(_("No such commit exists for this repository"), category="error")
             raise HTTPNotFound()
         except RepositoryError as e:
-            log.warning('Repository error while fetching filenode `%s`. Err:%s', path, e)
-            h.flash(h.escape(safe_str(e)), category='error')
+            log.warning("Repository error while fetching filenode `%s`. Err:%s", path, e)
+            h.flash(h.escape(safe_str(e)), category="error")
             raise HTTPNotFound()
 
         return file_node
 
     def _is_valid_head(self, commit_id, repo, landing_ref):
-        branch_name = sha_commit_id = ''
+        branch_name = sha_commit_id = ""
         is_head = False
-        log.debug('Checking if commit_id `%s` is a head for %s.', commit_id, repo)
+        log.debug("Checking if commit_id `%s` is a head for %s.", commit_id, repo)
 
         for _branch_name, branch_commit_id in repo.branches.items():
             # simple case we pass in branch name, it's a HEAD
             if commit_id == _branch_name:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
             # case when we pass in full sha commit_id, which is a head
             elif commit_id == branch_commit_id:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
 
         if h.is_svn(repo) and not repo.is_empty():
             # Note: Subversion only has one head.
             if commit_id == repo.get_commit(commit_idx=-1).raw_id:
                 is_head = True
                 return branch_name, sha_commit_id, is_head
 
         # checked branches, means we only need to try to get the branch/commit_sha
         if repo.is_empty():
             is_head = True
             branch_name = landing_ref
             sha_commit_id = EmptyCommit().raw_id
         else:
             commit = repo.get_commit(commit_id=commit_id)
             if commit:
                 branch_name = commit.branch
                 sha_commit_id = commit.raw_id
 
         return branch_name, sha_commit_id, is_head
 
     def _get_tree_at_commit(self, c, commit_id, f_path, full_load=False, at_rev=None):
-
         repo_id = self.db_repo.repo_id
         force_recache = self.get_recache_flag()
 
-        cache_seconds = safe_int(
-            rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
+        cache_seconds = rhodecode.ConfigGet().get_int("rc_cache.cache_repo.expiration_time")
         cache_on = not force_recache and cache_seconds > 0
+
         log.debug(
-            'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`'
-            'with caching: %s[TTL: %ss]' % (
-                repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
+            "Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`"
+            "with caching: %s[TTL: %ss]" % (repo_id, commit_id, f_path, cache_on, cache_seconds or 0)
+        )
 
-        cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
-        region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
+        cache_namespace_uid = f"repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}"
+        region = rc_cache.get_or_create_region("cache_repo", cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
         def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
-            log.debug('Generating cached file tree at for repo_id: %s, %s, %s',
-                      _repo_id, _commit_id, _f_path)
+            log.debug("Generating cached file tree at for repo_id: %s, %s, %s", _repo_id, _commit_id, _f_path)
 
             c.full_load = _full_load
             return render(
-                'rhodecode:templates/files/files_browser_tree.mako',
-                self._get_template_context(c), self.request, _at_rev)
+                "rhodecode:templates/files/files_browser_tree.mako",
+                self._get_template_context(c),
+                self.request,
+                _at_rev,
+            )
 
         return compute_file_tree(
-            self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
+            self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev
+        )
 
     def create_pure_path(self, *parts):
         # Split paths and sanitize them, removing any ../ etc
-        sanitized_path = [
-            x for x in pathlib.PurePath(*parts).parts
-            if x not in ['.', '..']]
+        sanitized_path = [x for x in pathlib.PurePath(*parts).parts if x not in [".", ".."]]
 
         pure_path = pathlib.PurePath(*sanitized_path)
         return pure_path
 
     def _is_lf_enabled(self, target_repo):
         lf_enabled = False
 
-        lf_key_for_vcs_map = {
-            'hg': 'extensions_largefiles',
-            'git': 'vcs_git_lfs_enabled'
-        }
+        lf_key_for_vcs_map = {"hg": "extensions_largefiles", "git": "vcs_git_lfs_enabled"}
 
         lf_key_for_vcs = lf_key_for_vcs_map.get(target_repo.repo_type)
 
         if lf_key_for_vcs:
             lf_enabled = self._get_repo_setting(target_repo, lf_key_for_vcs)
 
         return lf_enabled
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_archivefile(self):
         # archive cache config
         from rhodecode import CONFIG
+
         _ = self.request.translate
         self.load_default_context()
 
-        subrepos = self.request.GET.get('subrepos') == 'true'
-        with_hash = str2bool(self.request.GET.get('with_hash', '1'))
+        subrepos = self.request.GET.get("subrepos") == "true"
+        with_hash = str2bool(self.request.GET.get("with_hash", "1"))
 
-        default_at_path = '/'
-        fname = self.request.matchdict['fname']
-        at_path = self.request.GET.get('at_path') or default_at_path
+        default_at_path = "/"
+        fname = self.request.matchdict["fname"]
+        at_path = self.request.GET.get("at_path") or default_at_path
 
         if not self.db_repo.enable_downloads:
-            return Response(_('Downloads disabled'))
+            return Response(_("Downloads disabled"))
 
         try:
             commit_id, ext, file_format, content_type = _get_archive_spec(fname)
         except ValueError:
-            return Response(_('Unknown archive type for: `{}`').format(h.escape(fname)))
+            return Response(_("Unknown archive type for: `{}`").format(h.escape(fname)))
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id)
         except CommitDoesNotExistError:
-            return Response(_('Unknown commit_id {}').format(
-                h.escape(commit_id)))
+            return Response(_("Unknown commit_id {}").format(h.escape(commit_id)))
         except EmptyRepositoryError:
-            return Response(_('Empty repository'))
+            return Response(_("Empty repository"))
 
         # we used a ref, or a shorter version, lets redirect client ot use explicit hash
         if commit_id != commit.raw_id:
-            fname=f'{commit.raw_id}{ext}'
+            fname = f"{commit.raw_id}{ext}"
             raise HTTPFound(self.request.current_route_path(fname=fname))
 
         try:
             at_path = commit.get_node(safe_bytes(at_path)).path or default_at_path
         except Exception:
-            return Response(_('No node at path {} for this repository').format(h.escape(at_path)))
+            return Response(_("No node at path {} for this repository").format(h.escape(at_path)))
 
         path_sha = get_path_sha(at_path)
 
         # used for cache etc, consistent unique archive name
         archive_name_key = get_archive_name(
-            self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
-            path_sha=path_sha, with_hash=True)
+            self.db_repo.repo_id,
+            self.db_repo_name,
+            commit_sha=commit.short_id,
+            ext=ext,
+            subrepos=subrepos,
+            path_sha=path_sha,
+            with_hash=True,
+        )
 
         if not with_hash:
-            path_sha = ''
+            path_sha = ""
 
         # what end client gets served
         response_archive_name = get_archive_name(
-            self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
-            path_sha=path_sha, with_hash=with_hash)
+            self.db_repo.repo_id,
+            self.db_repo_name,
+            commit_sha=commit.short_id,
+            ext=ext,
+            subrepos=subrepos,
+            path_sha=path_sha,
+            with_hash=with_hash,
+        )
 
         # remove extension from our archive directory name
-        archive_dir_name = response_archive_name[:-len(ext)]
+        archive_dir_name = response_archive_name[: -len(ext)]
 
-        archive_cache_disable = self.request.GET.get('no_cache')
+        archive_cache_disable = self.request.GET.get("no_cache")
 
         d_cache = get_archival_cache_store(config=CONFIG)
 
         # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver
         d_cache_conf = get_archival_config(config=CONFIG)
 
         # This is also a cache key, and lock key
-        reentrant_lock_key = archive_name_key + '.lock'
+        reentrant_lock_key = archive_name_key + ".lock"
 
         use_cached_archive = False
         if not archive_cache_disable and archive_name_key in d_cache:
             reader, metadata = d_cache.fetch(archive_name_key)
 
             use_cached_archive = True
-            log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',
-                      archive_name_key, metadata, reader.name)
+            log.debug(
+                "Found cached archive as key=%s tag=%s, serving archive from cache reader=%s",
+                archive_name_key,
+                metadata,
+                reader.name,
+            )
         else:
             reader = None
-            log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key)
+            log.debug("Archive with key=%s is not yet cached, creating one now...", archive_name_key)
 
         if not reader:
             # generate new archive, as previous was not found in the cache
             try:
                 with d_cache.get_lock(reentrant_lock_key):
                     try:
-                        commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
-                                            kind=file_format, subrepos=subrepos,
-                                            archive_at_path=at_path, cache_config=d_cache_conf)
+                        commit.archive_repo(
+                            archive_name_key,
+                            archive_dir_name=archive_dir_name,
+                            kind=file_format,
+                            subrepos=subrepos,
+                            archive_at_path=at_path,
+                            cache_config=d_cache_conf,
+                        )
                     except ImproperArchiveTypeError:
-                        return _('Unknown archive type')
+                        return _("Unknown archive type")
 
             except ArchiveCacheGenerationLock:
                 retry_after = round(random.uniform(0.3, 3.0), 1)
                 time.sleep(retry_after)
 
                 location = self.request.url
                 response = Response(
                     f"archive {archive_name_key} generation in progress, Retry-After={retry_after}, Location={location}"
                 )
                 response.headers["Retry-After"] = str(retry_after)
                 response.status_code = 307  # temporary redirect
 
                 response.location = location
                 return response
 
             reader, metadata = d_cache.fetch(archive_name_key, retry=True, retry_attempts=30)
 
         response = Response(app_iter=archive_iterator(reader))
-        response.content_disposition = f'attachment; filename={response_archive_name}'
+        response.content_disposition = f"attachment; filename={response_archive_name}"
         response.content_type = str(content_type)
 
         try:
             return response
         finally:
             # store download action
             audit_logger.store_web(
-                'repo.archive.download', action_data={
-                    'user_agent': self.request.user_agent,
-                    'archive_name': archive_name_key,
-                    'archive_spec': fname,
-                    'archive_cached': use_cached_archive},
+                "repo.archive.download",
+                action_data={
+                    "user_agent": self.request.user_agent,
+                    "archive_name": archive_name_key,
+                    "archive_spec": fname,
+                    "archive_cached": use_cached_archive,
+                },
                 user=self._rhodecode_user,
                 repo=self.db_repo,
-                commit=True
+                commit=True,
             )
 
     def _get_file_node(self, commit_id, f_path):
-        if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
+        if commit_id not in ["", None, "None", "0" * 12, "0" * 40]:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             try:
                 node = commit.get_node(safe_bytes(f_path))
                 if node.is_dir():
-                    raise NodeError(f'{node} path is a {type(node)} not a file')
+                    raise NodeError(f"{node} path is a {type(node)} not a file")
             except NodeDoesNotExistError:
                 commit = EmptyCommit(
                     commit_id=commit_id,
                     idx=commit.idx,
                     repo=commit.repository,
                     alias=commit.repository.alias,
                     message=commit.message,
                     author=commit.author,
-                    date=commit.date)
-                node = FileNode(safe_bytes(f_path), b'', commit=commit)
+                    date=commit.date,
+                )
+                node = FileNode(safe_bytes(f_path), b"", commit=commit)
         else:
-            commit = EmptyCommit(
-                repo=self.rhodecode_vcs_repo,
-                alias=self.rhodecode_vcs_repo.alias)
-            node = FileNode(safe_bytes(f_path), b'', commit=commit)
+            commit = EmptyCommit(repo=self.rhodecode_vcs_repo, alias=self.rhodecode_vcs_repo.alias)
+            node = FileNode(safe_bytes(f_path), b"", commit=commit)
         return node
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_files_diff(self):
         c = self.load_default_context()
         f_path = self._get_f_path(self.request.matchdict)
-        diff1 = self.request.GET.get('diff1', '')
-        diff2 = self.request.GET.get('diff2', '')
+        diff1 = self.request.GET.get("diff1", "")
+        diff2 = self.request.GET.get("diff2", "")
 
         path1, diff1 = parse_path_ref(diff1, default_path=f_path)
 
-        ignore_whitespace = str2bool(self.request.GET.get('ignorews'))
-        line_context = self.request.GET.get('context', 3)
+        ignore_whitespace = str2bool(self.request.GET.get("ignorews"))
+        line_context = self.request.GET.get("context", 3)
 
         if not any((diff1, diff2)):
-            h.flash(
-                'Need query parameter "diff1" or "diff2" to generate a diff.',
-                category='error')
+            h.flash('Need query parameter "diff1" or "diff2" to generate a diff.', category="error")
             raise HTTPBadRequest()
 
-        c.action = self.request.GET.get('diff')
-        if c.action not in ['download', 'raw']:
+        c.action = self.request.GET.get("diff")
+        if c.action not in ["download", "raw"]:
             compare_url = h.route_path(
-                'repo_compare',
+                "repo_compare",
                 repo_name=self.db_repo_name,
-                source_ref_type='rev',
+                source_ref_type="rev",
                 source_ref=diff1,
                 target_repo=self.db_repo_name,
-                target_ref_type='rev',
+                target_ref_type="rev",
                 target_ref=diff2,
-                _query=dict(f_path=f_path))
+                _query=dict(f_path=f_path),
+            )
             # redirect to new view if we render diff
             raise HTTPFound(compare_url)
 
         try:
             node1 = self._get_file_node(diff1, path1)
             node2 = self._get_file_node(diff2, f_path)
         except (RepositoryError, NodeError):
             log.exception("Exception while trying to get node from repository")
-            raise HTTPFound(
-                h.route_path('repo_files', repo_name=self.db_repo_name,
-                             commit_id='tip', f_path=f_path))
+            raise HTTPFound(h.route_path("repo_files", repo_name=self.db_repo_name, commit_id="tip", f_path=f_path))
 
-        if all(isinstance(node.commit, EmptyCommit)
-               for node in (node1, node2)):
+        if all(isinstance(node.commit, EmptyCommit) for node in (node1, node2)):
             raise HTTPNotFound()
 
         c.commit_1 = node1.commit
         c.commit_2 = node2.commit
 
-        if c.action == 'download':
-            _diff = diffs.get_gitdiff(node1, node2,
-                                      ignore_whitespace=ignore_whitespace,
-                                      context=line_context)
+        if c.action == "download":
+            _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context)
             # NOTE: this was using diff_format='gitdiff'
-            diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
+            diff = diffs.DiffProcessor(_diff, diff_format="newdiff")
 
             response = Response(self.path_filter.get_raw_patch(diff))
-            response.content_type = 'text/plain'
-            response.content_disposition = (
-                f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff'
-            )
+            response.content_type = "text/plain"
+            response.content_disposition = f"attachment; filename={f_path}_{diff1}_vs_{diff2}.diff"
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
-        elif c.action == 'raw':
-            _diff = diffs.get_gitdiff(node1, node2,
-                                      ignore_whitespace=ignore_whitespace,
-                                      context=line_context)
+        elif c.action == "raw":
+            _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context)
             # NOTE: this was using diff_format='gitdiff'
-            diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
+            diff = diffs.DiffProcessor(_diff, diff_format="newdiff")
 
             response = Response(self.path_filter.get_raw_patch(diff))
-            response.content_type = 'text/plain'
+            response.content_type = "text/plain"
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
         # in case we ever end up here
         raise HTTPNotFound()
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_files_diff_2way_redirect(self):
         """
         Kept only to make OLD links work
         """
         f_path = self._get_f_path_unchecked(self.request.matchdict)
-        diff1 = self.request.GET.get('diff1', '')
-        diff2 = self.request.GET.get('diff2', '')
+        diff1 = self.request.GET.get("diff1", "")
+        diff2 = self.request.GET.get("diff2", "")
 
         if not any((diff1, diff2)):
-            h.flash(
-                'Need query parameter "diff1" or "diff2" to generate a diff.',
-                category='error')
+            h.flash('Need query parameter "diff1" or "diff2" to generate a diff.', category="error")
             raise HTTPBadRequest()
 
         compare_url = h.route_path(
-            'repo_compare',
+            "repo_compare",
             repo_name=self.db_repo_name,
-            source_ref_type='rev',
+            source_ref_type="rev",
             source_ref=diff1,
-            target_ref_type='rev',
+            target_ref_type="rev",
             target_ref=diff2,
-            _query=dict(f_path=f_path, diffmode='sideside',
-                        target_repo=self.db_repo_name,))
+            _query=dict(
+                f_path=f_path,
+                diffmode="sideside",
+                target_repo=self.db_repo_name,
+            ),
+        )
         raise HTTPFound(compare_url)
 
     @LoginRequired()
     def repo_files_default_commit_redirect(self):
         """
         Special page that redirects to the landing page of files based on the default
         commit for repository
         """
         c = self.load_default_context()
         ref_name = c.rhodecode_db_repo.landing_ref_name
         landing_url = h.repo_files_by_ref_url(
             c.rhodecode_db_repo.repo_name,
             c.rhodecode_db_repo.repo_type,
-            f_path='',
+            f_path="",
             ref_name=ref_name,
-            commit_id='tip',
-            query=dict(at=ref_name)
+            commit_id="tip",
+            query=dict(at=ref_name),
         )
 
         raise HTTPFound(landing_url)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_files(self):
         c = self.load_default_context()
 
-        view_name = getattr(self.request.matched_route, 'name', None)
+        view_name = getattr(self.request.matched_route, "name", None)
 
-        c.annotate = view_name == 'repo_files:annotated'
+        c.annotate = view_name == "repo_files:annotated"
         # default is false, but .rst/.md files later are auto rendered, we can
         # overwrite auto rendering by setting this GET flag
-        c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False)
+        c.renderer = view_name == "repo_files:rendered" or not self.request.GET.get("no-render", False)
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         c.commit = self._get_commit_or_redirect(commit_id)
-        c.branch = self.request.GET.get('branch', None)
+        c.branch = self.request.GET.get("branch", None)
         c.f_path = f_path
-        at_rev = self.request.GET.get('at')
+        at_rev = self.request.GET.get("at")
 
         # files or dirs
         try:
-
-            c.file = c.commit.get_node(bytes_path, pre_load=['is_binary', 'size', 'data'])
+            c.file = c.commit.get_node(bytes_path, pre_load=["is_binary", "size", "data"])
 
             c.file_author = True
-            c.file_tree = ''
+            c.file_tree = ""
 
             # prev link
             try:
                 prev_commit = c.commit.prev(c.branch)
                 c.prev_commit = prev_commit
-                c.url_prev = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path)
+                c.url_prev = h.route_path(
+                    "repo_files", repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path
+                )
                 if c.branch:
-                    c.url_prev += f'?branch={c.branch}'
+                    c.url_prev += f"?branch={c.branch}"
             except (CommitDoesNotExistError, VCSError):
-                c.url_prev = '#'
+                c.url_prev = "#"
                 c.prev_commit = EmptyCommit()
 
             # next link
             try:
                 next_commit = c.commit.next(c.branch)
                 c.next_commit = next_commit
-                c.url_next = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path)
+                c.url_next = h.route_path(
+                    "repo_files", repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path
+                )
                 if c.branch:
-                    c.url_next += f'?branch={c.branch}'
+                    c.url_next += f"?branch={c.branch}"
             except (CommitDoesNotExistError, VCSError):
-                c.url_next = '#'
+                c.url_next = "#"
                 c.next_commit = EmptyCommit()
 
             # load file content
             if c.file.is_file():
-
                 c.lf_node = {}
 
                 has_lf_enabled = self._is_lf_enabled(self.db_repo)
                 if has_lf_enabled:
                     c.lf_node = c.file.get_largefile_node()
 
-                c.file_source_page = 'true'
+                c.file_source_page = "true"
                 c.file_last_commit = c.file.last_commit
 
                 c.file_size_too_big = c.file.size > c.visual.cut_off_limit_file
 
                 if not (c.file_size_too_big or c.file.is_binary):
                     if c.annotate:  # annotation has precedence over renderer
-                        c.annotated_lines = filenode_as_annotated_lines_tokens(
-                            c.file
-                        )
+                        c.annotated_lines = filenode_as_annotated_lines_tokens(c.file)
                     else:
-                        c.renderer = (
-                            c.renderer and h.renderer_from_filename(c.file.path)
-                        )
+                        c.renderer = c.renderer and h.renderer_from_filename(c.file.path)
                         if not c.renderer:
                             c.lines = filenode_as_lines_tokens(c.file)
 
-                _branch_name, _sha_commit_id, is_head = \
-                    self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                        landing_ref=self.db_repo.landing_ref_name)
+                _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+                    commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+                )
                 c.on_branch_head = is_head
 
-                branch = c.commit.branch if (
-                    c.commit.branch and '/' not in c.commit.branch) else None
+                branch = c.commit.branch if (c.commit.branch and "/" not in c.commit.branch) else None
                 c.branch_or_raw_id = branch or c.commit.raw_id
                 c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id)
 
                 author = c.file_last_commit.author
-                c.authors = [[
-                    h.email(author),
-                    h.person(author, 'username_or_name_or_email'),
-                    1
-                ]]
+                c.authors = [[h.email(author), h.person(author, "username_or_name_or_email"), 1]]
 
-            else:  # load tree content at path
-                c.file_source_page = 'false'
+            else:  # load tree content (dir content) at path
+                c.file_source_page = "false"
                 c.authors = []
+
+                dir_node = c.file
+                c.file_nodes = dir_node.commit.get_nodes(dir_node.bytes_path, pre_load=dir_node.default_pre_load)
                 # this loads a simple tree without metadata to speed things up
                 # later via ajax we call repo_nodetree_full and fetch whole
                 c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev)
 
-                c.readme_data, c.readme_file = \
-                    self._get_readme_data(self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path)
+                c.readme_data, c.readme_file = self._get_readme_data(
+                    self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path, nodes=c.file_nodes
+                )
 
         except RepositoryError as e:
-            h.flash(h.escape(safe_str(e)), category='error')
+            h.flash(h.escape(safe_str(e)), category="error")
             raise HTTPNotFound()
 
-        if self.request.environ.get('HTTP_X_PJAX'):
-            html = render('rhodecode:templates/files/files_pjax.mako',
-                          self._get_template_context(c), self.request)
+        if self.request.environ.get("HTTP_X_PJAX"):
+            html = render("rhodecode:templates/files/files_pjax.mako", self._get_template_context(c), self.request)
         else:
-            html = render('rhodecode:templates/files/files.mako',
-                          self._get_template_context(c), self.request)
+            html = render("rhodecode:templates/files/files.mako", self._get_template_context(c), self.request)
         return Response(html)
 
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_files_annotated_previous(self):
         self.load_default_context()
 
         commit_id, bytes_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         prev_commit_id = commit.raw_id
-        line_anchor = self.request.GET.get('line_anchor')
+        line_anchor = self.request.GET.get("line_anchor")
         is_file = False
         try:
             _file = commit.get_node(bytes_path)
             is_file = _file.is_file()
         except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError):
             pass
 
         if is_file:
             history = commit.get_path_history(bytes_path)
-            prev_commit_id = history[1].raw_id \
-                if len(history) > 1 else prev_commit_id
+            prev_commit_id = history[1].raw_id if len(history) > 1 else prev_commit_id
         prev_url = h.route_path(
-            'repo_files:annotated', repo_name=self.db_repo_name,
-            commit_id=prev_commit_id, f_path=bytes_path,
-            _anchor=f'L{line_anchor}')
+            "repo_files:annotated",
+            repo_name=self.db_repo_name,
+            commit_id=prev_commit_id,
+            f_path=bytes_path,
+            _anchor=f"L{line_anchor}",
+        )
 
         raise HTTPFound(prev_url)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_nodetree_full(self):
         """
         Returns rendered html of file tree that contains commit date,
         author, commit_id for the specified combination of
         repo, commit_id and file path
         """
         c = self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         try:
             dir_node = commit.get_node(bytes_path)
         except RepositoryError as e:
-            return Response(f'error: {h.escape(safe_str(e))}')
+            return Response(f"error: {h.escape(safe_str(e))}")
 
         if dir_node.is_file():
-            return Response('')
+            return Response("")
 
         c.file = dir_node
+        c.file_nodes = dir_node.commit.get_nodes(dir_node.bytes_path, pre_load=dir_node.default_pre_load)
         c.commit = commit
-        at_rev = self.request.GET.get('at')
+        at_rev = self.request.GET.get("at")
 
-        html = self._get_tree_at_commit(
-            c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev)
+        html = self._get_tree_at_commit(c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev)
 
         return Response(html)
 
     def _get_attachement_headers(self, f_path):
         f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1])
         safe_path = f_name.replace('"', '\\"')
         encoded_path = urllib.parse.quote(f_name)
 
-        headers = f"attachment; " \
-                  f"filename=\"{safe_path}\"; " \
-                  f"filename*=UTF-8\'\'{encoded_path}"
+        headers = f"attachment; " f'filename="{safe_path}"; ' f"filename*=UTF-8''{encoded_path}"
 
         return header_safe_str(headers)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_file_raw(self):
         """
         Action for show as raw, some mimetypes are "rendered",
         those include images, icons.
         """
         c = self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, bytes_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
-
             # show images inline:
             # Do not re-add SVG: it is unsafe and permits XSS attacks. One can
             # for example render an SVG with javascript inside or even render
             # HTML.
-            'image/x-icon': ('image/x-icon', 'inline'),
-            'image/png': ('image/png', 'inline'),
-            'image/gif': ('image/gif', 'inline'),
-            'image/jpeg': ('image/jpeg', 'inline'),
-            'application/pdf': ('application/pdf', 'inline'),
+            "image/x-icon": ("image/x-icon", "inline"),
+            "image/png": ("image/png", "inline"),
+            "image/gif": ("image/gif", "inline"),
+            "image/jpeg": ("image/jpeg", "inline"),
+            "application/pdf": ("application/pdf", "inline"),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, disposition = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
-                mimetype, disposition = 'application/octet-stream', 'attachment'
+                mimetype, disposition = "application/octet-stream", "attachment"
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
-                mimetype, disposition = 'text/plain', 'inline'
+                mimetype, disposition = "text/plain", "inline"
 
-        if disposition == 'attachment':
+        if disposition == "attachment":
             disposition = self._get_attachement_headers(f_path)
 
         stream_content = file_node.stream_bytes()
 
         response = Response(app_iter=stream_content)
         response.content_disposition = disposition
         response.content_type = mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_file_download(self):
         c = self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, bytes_path)
 
-        if self.request.GET.get('lf'):
+        if self.request.GET.get("lf"):
             # only if lf get flag is passed, we download this file
             # as LFS/Largefile
             lf_node = file_node.get_largefile_node()
             if lf_node:
                 # overwrite our pointer with the REAL large-file
                 file_node = lf_node
 
         disposition = self._get_attachement_headers(f_path)
 
         stream_content = file_node.stream_bytes()
 
         response = Response(app_iter=stream_content)
         response.content_disposition = disposition
         response.content_type = file_node.mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path):
-
-        cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time')
+        cache_seconds = rhodecode.ConfigGet().get_int("rc_cache.cache_repo.expiration_time")
         cache_on = cache_seconds > 0
         log.debug(
-            'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`'
-            'with caching: %s[TTL: %ss]' % (
-                repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
+            "Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`"
+            "with caching: %s[TTL: %ss]" % (repo_id, commit_id, f_path, cache_on, cache_seconds or 0)
+        )
 
-        cache_namespace_uid = f'repo.{repo_id}'
-        region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
+        cache_namespace_uid = f"repo.{repo_id}"
+        region = rc_cache.get_or_create_region("cache_repo", cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
         def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path):
-            log.debug('Generating cached nodelist for repo_id:%s, %s, %s', _repo_id, commit_id, f_path)
+            log.debug("Generating cached nodelist for repo_id:%s, %s, %s", _repo_id, commit_id, f_path)
             try:
                 _d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path)
             except (RepositoryError, CommitDoesNotExistError, Exception) as e:
                 log.exception(safe_str(e))
-                h.flash(h.escape(safe_str(e)), category='error')
-                raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, commit_id='tip', f_path='/'))
+                h.flash(h.escape(safe_str(e)), category="error")
+                raise HTTPFound(h.route_path("repo_files", repo_name=self.db_repo_name, commit_id="tip", f_path="/"))
 
             return _d + _f
 
         result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path)
-        return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result)
+        return filter(lambda n: self.path_filter.path_access_allowed(n["name"]), result)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_nodelist(self):
         self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
 
-        metadata = self._get_nodelist_at_commit(
-            self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
-        return {'nodes': [x for x in metadata]}
+        metadata = self._get_nodelist_at_commit(self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
+        return {"nodes": [x for x in metadata]}
 
     def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type):
         items = []
         for name, commit_id in branches_or_tags.items():
             sym_ref = symbolic_reference(commit_id, name, f_path, ref_type)
             items.append((sym_ref, name, ref_type))
         return items
 
     def _symbolic_reference(self, commit_id, name, f_path, ref_type):
         return commit_id
 
     def _symbolic_reference_svn(self, commit_id, name, f_path, ref_type):
         return commit_id
 
         # NOTE(dan): old code we used in "diff" mode compare
         new_f_path = vcspath.join(name, f_path)
-        return f'{new_f_path}@{commit_id}'
+        return f"{new_f_path}@{commit_id}"
 
     def _get_node_history(self, commit_obj, f_path, commits=None):
         """
         get commit history for given node
 
         :param commit_obj: commit to calculate history
         :param f_path: path for node to calculate history for
         :param commits: if passed don't calculate history and take
             commits defined in this list
         """
         _ = self.request.translate
 
         # calculate history based on tip
         tip = self.rhodecode_vcs_repo.get_commit()
         if commits is None:
             pre_load = ["author", "branch"]
             try:
                 commits = tip.get_path_history(safe_bytes(f_path), pre_load=pre_load)
             except (NodeDoesNotExistError, CommitError):
                 # this node is not present at tip!
                 commits = commit_obj.get_path_history(safe_bytes(f_path), pre_load=pre_load)
 
         history = []
         commits_group = ([], _("Changesets"))
         for commit in commits:
-            branch = ' (%s)' % commit.branch if commit.branch else ''
-            n_desc = f'r{commit.idx}:{commit.short_id}{branch}'
-            commits_group[0].append((commit.raw_id, n_desc, 'sha'))
+            branch = " (%s)" % commit.branch if commit.branch else ""
+            n_desc = f"r{commit.idx}:{commit.short_id}{branch}"
+            commits_group[0].append((commit.raw_id, n_desc, "sha"))
         history.append(commits_group)
 
         symbolic_reference = self._symbolic_reference
 
-        if self.rhodecode_vcs_repo.alias == 'svn':
-            adjusted_f_path = RepoFilesView.adjust_file_path_for_svn(
-                f_path, self.rhodecode_vcs_repo)
+        if self.rhodecode_vcs_repo.alias == "svn":
+            adjusted_f_path = RepoFilesView.adjust_file_path_for_svn(f_path, self.rhodecode_vcs_repo)
             if adjusted_f_path != f_path:
                 log.debug(
-                    'Recognized svn tag or branch in file "%s", using svn '
-                    'specific symbolic references', f_path)
+                    'Recognized svn tag or branch in file "%s", using svn ' "specific symbolic references", f_path
+                )
                 f_path = adjusted_f_path
                 symbolic_reference = self._symbolic_reference_svn
 
-        branches = self._create_references(
-            self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, 'branch')
+        branches = self._create_references(self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, "branch")
         branches_group = (branches, _("Branches"))
 
-        tags = self._create_references(
-            self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, 'tag')
+        tags = self._create_references(self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, "tag")
         tags_group = (tags, _("Tags"))
 
         history.append(branches_group)
         history.append(tags_group)
 
         return history, commits
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_file_history(self):
         self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, bytes_path)
 
         if file_node.is_file():
             file_history, _hist = self._get_node_history(commit, f_path)
 
             res = []
             for section_items, section in file_history:
                 items = []
                 for obj_id, obj_text, obj_type in section_items:
-                    at_rev = ''
-                    if obj_type in ['branch', 'bookmark', 'tag']:
+                    at_rev = ""
+                    if obj_type in ["branch", "bookmark", "tag"]:
                         at_rev = obj_text
-                    entry = {
-                        'id': obj_id,
-                        'text': obj_text,
-                        'type': obj_type,
-                        'at_rev': at_rev
-                    }
+                    entry = {"id": obj_id, "text": obj_text, "type": obj_type, "at_rev": at_rev}
 
                     items.append(entry)
 
-                res.append({
-                    'text': section,
-                    'children': items
-                })
+                res.append({"text": section, "children": items})
 
-            data = {
-                'more': False,
-                'results': res
-            }
+            data = {"more": False, "results": res}
             return data
 
-        log.warning('Cannot fetch history for directory')
+        log.warning("Cannot fetch history for directory")
         raise HTTPBadRequest()
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator(
-        'repository.read', 'repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin")
     def repo_file_authors(self):
         c = self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, bytes_path)
 
         if not file_node.is_file():
             raise HTTPBadRequest()
 
         c.file_last_commit = file_node.last_commit
-        if self.request.GET.get('annotate') == '1':
+        if self.request.GET.get("annotate") == "1":
             # use _hist from annotation if annotation mode is on
             commit_ids = {x[1] for x in file_node.annotate}
-            _hist = (
-                self.rhodecode_vcs_repo.get_commit(commit_id)
-                for commit_id in commit_ids)
+            _hist = (self.rhodecode_vcs_repo.get_commit(commit_id) for commit_id in commit_ids)
         else:
             _f_history, _hist = self._get_node_history(commit, f_path)
         c.file_author = False
 
         unique = collections.OrderedDict()
         for commit in _hist:
             author = commit.author
             if author not in unique:
                 unique[commit.author] = [
                     h.email(author),
-                    h.person(author, 'username_or_name_or_email'),
-                    1  # counter
+                    h.person(author, "username_or_name_or_email"),
+                    1,  # counter
                 ]
 
             else:
                 # increase counter
                 unique[commit.author][2] += 1
 
         c.authors = [val for val in unique.values()]
 
         return self._get_template_context(c)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     def repo_files_check_head(self):
         self.load_default_context()
 
         commit_id, f_path, bytes_path = self._get_commit_and_path()
-        _branch_name, _sha_commit_id, is_head = \
-            self._is_valid_head(commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name)
+        _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+            commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+        )
 
-        new_path = self.request.POST.get('path')
-        operation = self.request.POST.get('operation')
-        path_exist = ''
+        new_path = self.request.POST.get("path")
+        operation = self.request.POST.get("operation")
+        path_exist = ""
 
-        if new_path and operation in ['create', 'upload']:
-            new_f_path = os.path.join(f_path.lstrip('/'), new_path)
+        if new_path and operation in ["create", "upload"]:
+            new_f_path = os.path.join(f_path.lstrip("/"), new_path)
             try:
                 commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id)
                 # NOTE(dan): construct whole path without leading /
                 file_node = commit_obj.get_node(safe_bytes(new_f_path))
                 if file_node:
                     path_exist = new_f_path
             except (EmptyRepositoryError, NodeDoesNotExistError):
                 pass
 
-        return {
-            'branch': _branch_name,
-            'sha': _sha_commit_id,
-            'is_head': is_head,
-            'path_exists': path_exist
-        }
+        return {"branch": _branch_name, "sha": _sha_commit_id, "is_head": is_head, "path_exists": path_exist}
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     def repo_files_remove_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
-        _branch_name, _sha_commit_id, is_head = \
-            self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                landing_ref=self.db_repo.landing_ref_name)
+        _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+            commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+        )
 
         self.forbid_non_head(is_head, f_path)
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
 
-        c.default_message = _(
-            'Deleted file {} via RhodeCode Enterprise').format(f_path)
+        c.default_message = _("Deleted file {} via RhodeCode Enterprise").format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     @CSRFRequired()
     def repo_files_delete_file(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
-        _branch_name, _sha_commit_id, is_head = \
-            self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                landing_ref=self.db_repo.landing_ref_name)
+        _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+            commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+        )
 
         self.forbid_non_head(is_head, f_path)
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
 
-        c.default_message = _('Deleted file {} via RhodeCode Enterprise').format(f_path)
+        c.default_message = _("Deleted file {} via RhodeCode Enterprise").format(f_path)
         c.f_path = f_path
         node_path = f_path
         author = self._rhodecode_db_user.full_contact
-        message = self.request.POST.get('message') or c.default_message
+        message = self.request.POST.get("message") or c.default_message
         try:
-            nodes = {
-                safe_bytes(node_path): {
-                    'content': b''
-                }
-            }
+            nodes = {safe_bytes(node_path): {"content": b""}}
             ScmModel().delete_nodes(
-                user=self._rhodecode_db_user.user_id, repo=self.db_repo,
+                user=self._rhodecode_db_user.user_id,
+                repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
-            h.flash(
-                _('Successfully deleted file `{}`').format(
-                    h.escape(f_path)), category='success')
+            h.flash(_("Successfully deleted file `{}`").format(h.escape(f_path)), category="success")
         except Exception:
-            log.exception('Error during commit operation')
-            h.flash(_('Error occurred during commit'), category='error')
-        raise HTTPFound(
-            h.route_path('repo_commit', repo_name=self.db_repo_name,
-                         commit_id='tip'))
+            log.exception("Error during commit operation")
+            h.flash(_("Error occurred during commit"), category="error")
+        raise HTTPFound(h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip"))
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     def repo_files_edit_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
-        _branch_name, _sha_commit_id, is_head = \
-            self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                landing_ref=self.db_repo.landing_ref_name)
+        _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+            commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+        )
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
 
         if c.file.is_binary:
             files_url = h.route_path(
-                'repo_files',
-                repo_name=self.db_repo_name,
-                commit_id=c.commit.raw_id, f_path=f_path)
+                "repo_files", repo_name=self.db_repo_name, commit_id=c.commit.raw_id, f_path=f_path
+            )
             raise HTTPFound(files_url)
 
-        c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path)
+        c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     @CSRFRequired()
     def repo_files_update_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
 
         if c.file.is_binary:
-            raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name,
-                                         commit_id=c.commit.raw_id, f_path=f_path))
+            raise HTTPFound(
+                h.route_path("repo_files", repo_name=self.db_repo_name, commit_id=c.commit.raw_id, f_path=f_path)
+            )
 
-        _branch_name, _sha_commit_id, is_head = \
-            self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                landing_ref=self.db_repo.landing_ref_name)
+        _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+            commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+        )
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
-        c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path)
+        c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path)
         c.f_path = f_path
 
         old_content = c.file.str_content
         sl = old_content.splitlines(1)
-        first_line = sl[0] if sl else ''
+        first_line = sl[0] if sl else ""
 
         r_post = self.request.POST
         # line endings:  0 - Unix, 1 - Mac, 2 - DOS
         line_ending_mode = detect_mode(first_line, 0)
-        content = convert_line_endings(r_post.get('content', ''), line_ending_mode)
+        content = convert_line_endings(r_post.get("content", ""), line_ending_mode)
 
-        message = r_post.get('message') or c.default_message
+        message = r_post.get("message") or c.default_message
 
         org_node_path = c.file.str_path
-        filename = r_post['filename']
+        filename = r_post["filename"]
 
         root_path = c.file.dir_path
         pure_path = self.create_pure_path(root_path, filename)
         node_path = pure_path.as_posix()
 
-        default_redirect_url = h.route_path('repo_commit', repo_name=self.db_repo_name,
-                                            commit_id=commit_id)
+        default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit_id)
         if content == old_content and node_path == org_node_path:
-            h.flash(_('No changes detected on {}').format(h.escape(org_node_path)),
-                    category='warning')
+            h.flash(_("No changes detected on {}").format(h.escape(org_node_path)), category="warning")
             raise HTTPFound(default_redirect_url)
 
         try:
             mapping = {
                 c.file.bytes_path: {
-                    'org_filename': org_node_path,
-                    'filename': safe_bytes(node_path),
-                    'content': safe_bytes(content),
-                    'lexer': '',
-                    'op': 'mod',
-                    'mode': c.file.mode
+                    "org_filename": org_node_path,
+                    "filename": safe_bytes(node_path),
+                    "content": safe_bytes(content),
+                    "lexer": "",
+                    "op": "mod",
+                    "mode": c.file.mode,
                 }
             }
 
             commit = ScmModel().update_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=mapping,
                 parent_commit=c.commit,
             )
 
-            h.flash(_('Successfully committed changes to file `{}`').format(
-                    h.escape(f_path)), category='success')
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
+            h.flash(_("Successfully committed changes to file `{}`").format(h.escape(f_path)), category="success")
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except Exception:
-            log.exception('Error occurred during commit')
-            h.flash(_('Error occurred during commit'), category='error')
+            log.exception("Error occurred during commit")
+            h.flash(_("Error occurred during commit"), category="error")
 
         raise HTTPFound(default_redirect_url)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     def repo_files_add_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         # Check if we need to use this page to upload binary
-        upload_binary = str2bool(self.request.params.get('upload_binary', False))
+        upload_binary = str2bool(self.request.params.get("upload_binary", False))
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
-            _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
+            _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True
         else:
-            _branch_name, _sha_commit_id, is_head = \
-                self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                    landing_ref=self.db_repo.landing_ref_name)
+            _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+                commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+            )
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
-        c.default_message = (_('Added file via RhodeCode Enterprise')) \
-            if not upload_binary else (_('Edited file {} via RhodeCode Enterprise').format(f_path))
-        c.f_path = f_path.lstrip('/')  # ensure not relative path
+        c.default_message = (
+            (_("Added file via RhodeCode Enterprise"))
+            if not upload_binary
+            else (_("Edited file {} via RhodeCode Enterprise").format(f_path))
+        )
+        c.f_path = f_path.lstrip("/")  # ensure not relative path
         c.replace_binary = upload_binary
 
         return self._get_template_context(c)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     @CSRFRequired()
     def repo_files_create_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         # calculate redirect URL
         if self.rhodecode_vcs_repo.is_empty():
-            default_redirect_url = h.route_path(
-                'repo_summary', repo_name=self.db_repo_name)
+            default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name)
         else:
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip")
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
-            _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
+            _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True
         else:
-            _branch_name, _sha_commit_id, is_head = \
-                self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                    landing_ref=self.db_repo.landing_ref_name)
+            _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+                commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+            )
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
-        c.default_message = (_('Added file via RhodeCode Enterprise'))
+        c.default_message = _("Added file via RhodeCode Enterprise")
         c.f_path = f_path
 
         r_post = self.request.POST
-        message = r_post.get('message') or c.default_message
-        filename = r_post.get('filename')
+        message = r_post.get("message") or c.default_message
+        filename = r_post.get("filename")
         unix_mode = 0
 
         if not filename:
             # If there's no commit, redirect to repo summary
             if type(c.commit) is EmptyCommit:
-                redirect_url = h.route_path(
-                    'repo_summary', repo_name=self.db_repo_name)
+                redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name)
             else:
                 redirect_url = default_redirect_url
-            h.flash(_('No filename specified'), category='warning')
+            h.flash(_("No filename specified"), category="warning")
             raise HTTPFound(redirect_url)
 
         root_path = f_path
         pure_path = self.create_pure_path(root_path, filename)
-        node_path = pure_path.as_posix().lstrip('/')
+        node_path = pure_path.as_posix().lstrip("/")
 
         author = self._rhodecode_db_user.full_contact
-        content = convert_line_endings(r_post.get('content', ''), unix_mode)
-        nodes = {
-            safe_bytes(node_path): {
-                'content': safe_bytes(content)
-            }
-        }
+        content = convert_line_endings(r_post.get("content", ""), unix_mode)
+        nodes = {safe_bytes(node_path): {"content": safe_bytes(content)}}
 
         try:
-
             commit = ScmModel().create_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
-            h.flash(_('Successfully committed new file `{}`').format(h.escape(node_path)), category='success')
+            h.flash(_("Successfully committed new file `{}`").format(h.escape(node_path)), category="success")
 
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except NonRelativePathError:
-            log.exception('Non Relative path found')
-            h.flash(_('The location specified must be a relative path and must not '
-                      'contain .. in the path'), category='warning')
+            log.exception("Non Relative path found")
+            h.flash(
+                _("The location specified must be a relative path and must not " "contain .. in the path"),
+                category="warning",
+            )
             raise HTTPFound(default_redirect_url)
         except (NodeError, NodeAlreadyExistsError) as e:
-            h.flash(h.escape(safe_str(e)), category='error')
+            h.flash(h.escape(safe_str(e)), category="error")
         except Exception:
-            log.exception('Error occurred during commit')
-            h.flash(_('Error occurred during commit'), category='error')
+            log.exception("Error occurred during commit")
+            h.flash(_("Error occurred during commit"), category="error")
 
         raise HTTPFound(default_redirect_url)
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     @CSRFRequired()
     def repo_files_upload_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         # calculate redirect URL
         if self.rhodecode_vcs_repo.is_empty():
-            default_redirect_url = h.route_path(
-                'repo_summary', repo_name=self.db_repo_name)
+            default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name)
         else:
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip")
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
-            _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
+            _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True
         else:
-            _branch_name, _sha_commit_id, is_head = \
-                self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                    landing_ref=self.db_repo.landing_ref_name)
+            _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+                commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+            )
 
         error = self.forbid_non_head(is_head, f_path, json_mode=True)
         if error:
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
         error = self.check_branch_permission(_branch_name, json_mode=True)
         if error:
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
 
-        c.default_message = (_('Added file via RhodeCode Enterprise'))
+        c.default_message = _("Added file via RhodeCode Enterprise")
         c.f_path = f_path
 
         r_post = self.request.POST
 
         message = c.default_message
-        user_message = r_post.getall('message')
+        user_message = r_post.getall("message")
         if isinstance(user_message, list) and user_message:
             # we take the first from duplicated results if it's not empty
             message = user_message[0] if user_message[0] else message
 
         nodes = {}
 
-        for file_obj in r_post.getall('files_upload') or []:
+        for file_obj in r_post.getall("files_upload") or []:
             content = file_obj.file
             filename = file_obj.filename
 
             root_path = f_path
             pure_path = self.create_pure_path(root_path, filename)
-            node_path = pure_path.as_posix().lstrip('/')
+            node_path = pure_path.as_posix().lstrip("/")
 
-            nodes[safe_bytes(node_path)] = {
-                'content': content
-            }
+            nodes[safe_bytes(node_path)] = {"content": content}
 
         if not nodes:
-            error = 'missing files'
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            error = "missing files"
+            return {"error": error, "redirect_url": default_redirect_url}
 
         author = self._rhodecode_db_user.full_contact
 
         try:
             commit = ScmModel().create_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
             if len(nodes) == 1:
-                flash_message = _('Successfully committed {} new files').format(len(nodes))
+                flash_message = _("Successfully committed {} new files").format(len(nodes))
             else:
-                flash_message = _('Successfully committed 1 new file')
+                flash_message = _("Successfully committed 1 new file")
 
-            h.flash(flash_message, category='success')
+            h.flash(flash_message, category="success")
 
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except NonRelativePathError:
-            log.exception('Non Relative path found')
-            error = _('The location specified must be a relative path and must not '
-                      'contain .. in the path')
-            h.flash(error, category='warning')
+            log.exception("Non Relative path found")
+            error = _("The location specified must be a relative path and must not " "contain .. in the path")
+            h.flash(error, category="warning")
 
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
         except (NodeError, NodeAlreadyExistsError) as e:
             error = h.escape(e)
-            h.flash(error, category='error')
+            h.flash(error, category="error")
 
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
         except Exception:
-            log.exception('Error occurred during commit')
-            error = _('Error occurred during commit')
-            h.flash(error, category='error')
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            log.exception("Error occurred during commit")
+            error = _("Error occurred during commit")
+            h.flash(error, category="error")
+            return {"error": error, "redirect_url": default_redirect_url}
 
-        return {
-            'error': None,
-            'redirect_url': default_redirect_url
-        }
+        return {"error": None, "redirect_url": default_redirect_url}
 
     @LoginRequired()
-    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @HasRepoPermissionAnyDecorator("repository.write", "repository.admin")
     @CSRFRequired()
     def repo_files_replace_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path, bytes_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         if self.rhodecode_vcs_repo.is_empty():
-            default_redirect_url = h.route_path(
-                'repo_summary', repo_name=self.db_repo_name)
+            default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name)
         else:
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip")
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
-            _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
+            _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True
         else:
-            _branch_name, _sha_commit_id, is_head = \
-                self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
-                                    landing_ref=self.db_repo.landing_ref_name)
+            _branch_name, _sha_commit_id, is_head = self._is_valid_head(
+                commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name
+            )
 
         error = self.forbid_non_head(is_head, f_path, json_mode=True)
         if error:
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
         error = self.check_branch_permission(_branch_name, json_mode=True)
         if error:
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
 
-        c.default_message = (_('Edited file {} via RhodeCode Enterprise').format(f_path))
+        c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path)
         c.f_path = f_path
 
         r_post = self.request.POST
 
         message = c.default_message
-        user_message = r_post.getall('message')
+        user_message = r_post.getall("message")
         if isinstance(user_message, list) and user_message:
             # we take the first from duplicated results if it's not empty
             message = user_message[0] if user_message[0] else message
 
-        data_for_replacement = r_post.getall('files_upload') or []
+        data_for_replacement = r_post.getall("files_upload") or []
         if (objects_count := len(data_for_replacement)) > 1:
-            return {
-                'error': 'too many files for replacement',
-                'redirect_url': default_redirect_url
-            }
+            return {"error": "too many files for replacement", "redirect_url": default_redirect_url}
         elif not objects_count:
-            return {
-                'error': 'missing files',
-                'redirect_url': default_redirect_url
-            }
+            return {"error": "missing files", "redirect_url": default_redirect_url}
 
         content = data_for_replacement[0].file
         retrieved_filename = data_for_replacement[0].filename
 
-        if retrieved_filename.split('.')[-1] != f_path.split('.')[-1]:
+        if retrieved_filename.split(".")[-1] != f_path.split(".")[-1]:
             return {
-                'error': 'file extension of uploaded file doesn\'t match an original file\'s extension',
-                'redirect_url': default_redirect_url
+                "error": "file extension of uploaded file doesn't match an original file's extension",
+                "redirect_url": default_redirect_url,
             }
 
         author = self._rhodecode_db_user.full_contact
 
         try:
             commit = ScmModel().update_binary_node(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 node={
-                    'content': content,
-                    'file_path': f_path.encode(),
+                    "content": content,
+                    "file_path": f_path.encode(),
                 },
                 parent_commit=c.commit,
                 author=author,
             )
 
-            h.flash(_('Successfully committed 1 new file'), category='success')
+            h.flash(_("Successfully committed 1 new file"), category="success")
 
-            default_redirect_url = h.route_path(
-                'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
+            default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except (NodeError, NodeAlreadyExistsError) as e:
             error = h.escape(e)
-            h.flash(error, category='error')
+            h.flash(error, category="error")
 
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            return {"error": error, "redirect_url": default_redirect_url}
         except Exception:
-            log.exception('Error occurred during commit')
-            error = _('Error occurred during commit')
-            h.flash(error, category='error')
-            return {
-                'error': error,
-                'redirect_url': default_redirect_url
-            }
+            log.exception("Error occurred during commit")
+            error = _("Error occurred during commit")
+            h.flash(error, category="error")
+            return {"error": error, "redirect_url": default_redirect_url}
 
-        return {
-            'error': None,
-            'redirect_url': default_redirect_url
-        }
+        return {"error": None, "redirect_url": default_redirect_url}

rhodecode/lib/rc_cache/__init__.py

@@ -1,119 +1,119
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import threading
 
 from dogpile.cache import register_backend
 
 from . import region_meta
 from .utils import (
     ActiveRegionCache,
     InvalidationContext,
     backend_key_generator,
     clear_cache_namespace,
     get_default_cache_settings,
     get_or_create_region,
     make_region,
     str2bool,
 )
 
 module_name = 'rhodecode'
 
 register_backend(
     "dogpile.cache.rc.memory_lru", f"{module_name}.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
 register_backend(
     "dogpile.cache.rc.file_namespace", f"{module_name}.lib.rc_cache.backends",
     "FileNamespaceBackend")
 
 register_backend(
     "dogpile.cache.rc.redis", f"{module_name}.lib.rc_cache.backends",
     "RedisPickleBackend")
 
 register_backend(
     "dogpile.cache.rc.redis_msgpack", f"{module_name}.lib.rc_cache.backends",
     "RedisMsgPackBackend")
 
 
 log = logging.getLogger(__name__)
 
 
-FILE_TREE_CACHE_VER = 'v5'
+FILE_TREE_CACHE_VER = 'v6'
 LICENSE_CACHE_VER = 'v3'
 PERMISSIONS_CACHE_VER = 'v2'
 
 CLEAR_DELETE = 'delete'
 CLEAR_INVALIDATE = 'invalidate'
 
 
 def async_creation_runner(cache, cache_key, creator, mutex):
 
     def runner():
         try:
             value = creator()
             cache.set(cache_key, value)
         finally:
             mutex.release()
 
     thread = threading.Thread(target=runner)
     thread.start()
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         if namespace_name in avail_regions:
             continue
 
         avail_regions.add(namespace_name)
         log.debug('dogpile: found following cache regions: %s', namespace_name)
 
         new_region = make_region(
             name=namespace_name,
             function_key_generator=None,
             async_creation_runner=None
         )
 
         new_region.configure_from_config(settings, f'rc_cache.{namespace_name}.')
         new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
 
         async_creator = str2bool(settings.pop(f'rc_cache.{namespace_name}.async_creator', 'false'))
         if async_creator:
             log.debug('configuring region %s with async creator', new_region)
             new_region.async_creation_runner = async_creation_runner
 
         if log.isEnabledFor(logging.DEBUG):
             region_args = dict(backend=new_region.actual_backend,
                                region_invalidator=new_region.region_invalidator.__class__)
             log.debug('dogpile: registering a new region key=`%s` args=%s', namespace_name, region_args)
 
         region_meta.dogpile_cache_regions[namespace_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

rhodecode/lib/utils2.py

@@ -1,987 +1,988
 # Copyright (C) 2011-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Some simple helper functions
 """
 
 import collections
 import datetime
 import dateutil.relativedelta
 import logging
 import re
 import sys
 import time
 import urllib.request
 import urllib.parse
 import urllib.error
 import urlobject
 import uuid
 import getpass
 import socket
 import errno
 import random
 import functools
 from contextlib import closing
 
 import pygments.lexers
 import sqlalchemy
 import sqlalchemy.event
 import sqlalchemy.engine.url
 import sqlalchemy.exc
 import sqlalchemy.sql
 import webob
 from pyramid.settings import asbool
 
 import rhodecode
 from rhodecode.translation import _, _pluralize
 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
 from rhodecode.lib.hash_utils import md5, md5_safe, sha1, sha1_safe
 from rhodecode.lib.type_utils import aslist, str2bool, StrictAttributeDict, AttributeDict
 
 
 def __get_lem(extra_mapping=None):
     """
     Get language extension map based on what's inside pygments lexers
     """
     d = collections.defaultdict(lambda: [])
 
     def __clean(s):
         s = s.lstrip('*')
         s = s.lstrip('.')
 
         if s.find('[') != -1:
             exts = []
             start, stop = s.find('['), s.find(']')
 
             for suffix in s[start + 1:stop]:
                 exts.append(s[:s.find('[')] + suffix)
             return [e.lower() for e in exts]
         else:
             return [s.lower()]
 
     for lx, t in sorted(pygments.lexers.LEXERS.items()):
         m = list(map(__clean, t[-2]))
         if m:
             m = functools.reduce(lambda x, y: x + y, m)
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
 
     data = dict(d)
 
     extra_mapping = extra_mapping or {}
     if extra_mapping:
         for k, v in list(extra_mapping.items()):
             if k not in data:
                 # register new mapping2lexer
                 data[k] = [v]
 
     return data
 
 
 def convert_line_endings(line: str, mode) -> str:
     """
     Converts a given line  "line end" accordingly to given mode
 
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
 
     :param line: given line to convert
     :param mode: mode to convert to
     :return: converted line according to mode
     """
     if mode == 0:
         line = line.replace('\r\n', '\n')
         line = line.replace('\r', '\n')
     elif mode == 1:
         line = line.replace('\r\n', '\r')
         line = line.replace('\n', '\r')
     elif mode == 2:
         line = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', line)
     return line
 
 
 def detect_mode(line: str, default) -> int:
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def remove_suffix(s, suffix):
     if s.endswith(suffix):
         s = s[:-1 * len(suffix)]
     return s
 
 
 def remove_prefix(s, prefix):
     if s.startswith(prefix):
         s = s[len(prefix):]
     return s
 
 
 def find_calling_context(ignore_modules=None, depth=4, output_writer=None, indent=True):
     """
+    How to find calling context:
     Look through the calling stack and return the frame which called
     this function and is part of core module ( ie. rhodecode.* )
 
     :param ignore_modules: list of modules to ignore eg. ['rhodecode.lib']
     :param depth:
     :param output_writer:
     :param indent:
 
     usage::
 
         from rhodecode.lib.utils2 import find_calling_context
 
         calling_context = find_calling_context(ignore_modules=[
             'rhodecode.lib.caching_query',
             'rhodecode.model.settings',
         ])
 
     """
     import inspect
     if not output_writer:
         try:
             from rich import print as pprint
         except ImportError:
             pprint = print
         output_writer = pprint
 
     frame = inspect.currentframe()
     cc = []
     try:
         for i in range(depth):  # current frame + 3 callers
             frame = frame.f_back
             if not frame:
                 break
 
             info = inspect.getframeinfo(frame)
             name = frame.f_globals.get('__name__')
             if name not in ignore_modules:
                 cc.insert(0, f'CALL_CONTEXT:{i}: file {info.filename}:{info.lineno} -> {info.function}')
     finally:
         # Avoids a reference cycle
         del frame
 
     output_writer('* INFO: This code was called from: *')
     for cnt, frm_info in enumerate(cc):
         if not indent:
             cnt = 1
         output_writer(' ' * cnt + frm_info)
 
 
 def ping_connection(connection, branch):
     if branch:
         # "branch" refers to a sub-connection of a connection,
         # we don't want to bother pinging on these.
         return
 
     # turn off "close with result".  This flag is only used with
     # "connectionless" execution, otherwise will be False in any case
     save_should_close_with_result = connection.should_close_with_result
     connection.should_close_with_result = False
 
     try:
         # run a SELECT 1.   use a core select() so that
         # the SELECT of a scalar value without a table is
         # appropriately formatted for the backend
         connection.scalar(sqlalchemy.sql.select([1]))
     except sqlalchemy.exc.DBAPIError as err:
         # catch SQLAlchemy's DBAPIError, which is a wrapper
         # for the DBAPI's exception.  It includes a .connection_invalidated
         # attribute which specifies if this connection is a "disconnect"
         # condition, which is based on inspection of the original exception
         # by the dialect in use.
         if err.connection_invalidated:
             # run the same SELECT again - the connection will re-validate
             # itself and establish a new connection.  The disconnect detection
             # here also causes the whole connection pool to be invalidated
             # so that all stale connections are discarded.
             connection.scalar(sqlalchemy.sql.select([1]))
         else:
             raise
     finally:
         # restore "close with result"
         connection.should_close_with_result = save_should_close_with_result
 
 
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """Custom engine_from_config functions."""
     log = logging.getLogger('sqlalchemy.engine')
     use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None))
     debug = asbool(configuration.pop('sqlalchemy.db1.debug_query', None))
 
     engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs)
 
     def color_sql(sql):
         color_seq = '\033[1;33m'  # This is yellow: code 33
         normal = '\x1b[0m'
         return ''.join([color_seq, sql, normal])
 
     if use_ping_connection:
         log.debug('Adding ping_connection on the engine config.')
         sqlalchemy.event.listen(engine, "engine_connect", ping_connection)
 
     if debug:
         # attach events only for debug configuration
         def before_cursor_execute(conn, cursor, statement,
                                   parameters, context, executemany):
             setattr(conn, 'query_start_time', time.time())
             log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
             find_calling_context(ignore_modules=[
                 'rhodecode.lib.caching_query',
                 'rhodecode.model.settings',
             ], output_writer=log.info)
 
         def after_cursor_execute(conn, cursor, statement,
                                  parameters, context, executemany):
             delattr(conn, 'query_start_time')
 
         sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute)
         sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute)
 
     return engine
 
 
 def get_encryption_key(config) -> bytes:
     secret = config.get('rhodecode.encrypted_values.secret')
     default = config['beaker.session.secret']
     enc_key = secret or default
 
     return safe_bytes(enc_key)
 
 
 def age(prevdate, now=None, show_short_version=False, show_suffix=True, short_format=False):
     """
     Turns a datetime into an age string.
     If show_short_version is True, this generates a shorter string with
     an approximate age; ex. '1 day ago', rather than '1 day and 23 hours ago'.
 
     * IMPORTANT*
     Code of this function is written in special way so it's easier to
     backport it to javascript. If you mean to update it, please also update
     `jquery.timeago-extension.js` file
 
     :param prevdate: datetime object
     :param now: get current time, if not define we use
         `datetime.datetime.now()`
     :param show_short_version: if it should approximate the date and
         return a shorter string
     :param show_suffix:
     :param short_format: show short format, eg 2D instead of 2 days
     :rtype: unicode
     :returns: unicode words describing age
     """
 
     def _get_relative_delta(now, prevdate):
         base = dateutil.relativedelta.relativedelta(now, prevdate)
         return {
             'year': base.years,
             'month': base.months,
             'day': base.days,
             'hour': base.hours,
             'minute': base.minutes,
             'second': base.seconds,
         }
 
     def _is_leap_year(year):
         return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
 
     def get_month(prevdate):
         return prevdate.month
 
     def get_year(prevdate):
         return prevdate.year
 
     now = now or datetime.datetime.now()
     order = ['year', 'month', 'day', 'hour', 'minute', 'second']
     deltas = {}
     future = False
 
     if prevdate > now:
         now_old = now
         now = prevdate
         prevdate = now_old
         future = True
     if future:
         prevdate = prevdate.replace(microsecond=0)
     # Get date parts deltas
     for part in order:
         rel_delta = _get_relative_delta(now, prevdate)
         deltas[part] = rel_delta[part]
 
     # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
     # not 1 hour, -59 minutes and -59 seconds)
     offsets = [[5, 60], [4, 60], [3, 24]]
     for element in offsets:  # seconds, minutes, hours
         num = element[0]
         length = element[1]
 
         part = order[num]
         carry_part = order[num - 1]
 
         if deltas[part] < 0:
             deltas[part] += length
             deltas[carry_part] -= 1
 
     # Same thing for days except that the increment depends on the (variable)
     # number of days in the month
     month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
     if deltas['day'] < 0:
         if get_month(prevdate) == 2 and _is_leap_year(get_year(prevdate)):
             deltas['day'] += 29
         else:
             deltas['day'] += month_lengths[get_month(prevdate) - 1]
 
         deltas['month'] -= 1
 
     if deltas['month'] < 0:
         deltas['month'] += 12
         deltas['year'] -= 1
 
     # Format the result
     if short_format:
         fmt_funcs = {
             'year': lambda d: '%dy' % d,
             'month': lambda d: '%dm' % d,
             'day': lambda d: '%dd' % d,
             'hour': lambda d: '%dh' % d,
             'minute': lambda d: '%dmin' % d,
             'second': lambda d: '%dsec' % d,
         }
     else:
         fmt_funcs = {
             'year': lambda d: _pluralize('${num} year', '${num} years', d, mapping={'num': d}).interpolate(),
             'month': lambda d: _pluralize('${num} month', '${num} months', d, mapping={'num': d}).interpolate(),
             'day': lambda d: _pluralize('${num} day', '${num} days', d, mapping={'num': d}).interpolate(),
             'hour': lambda d: _pluralize('${num} hour', '${num} hours', d, mapping={'num': d}).interpolate(),
             'minute': lambda d: _pluralize('${num} minute', '${num} minutes', d, mapping={'num': d}).interpolate(),
             'second': lambda d: _pluralize('${num} second', '${num} seconds', d, mapping={'num': d}).interpolate(),
         }
 
     i = 0
     for part in order:
         value = deltas[part]
         if value != 0:
 
             if i < 5:
                 sub_part = order[i + 1]
                 sub_value = deltas[sub_part]
             else:
                 sub_value = 0
 
             if sub_value == 0 or show_short_version:
                 _val = fmt_funcs[part](value)
                 if future:
                     if show_suffix:
                         return _('in ${ago}', mapping={'ago': _val})
                     else:
                         return _(_val)
 
                 else:
                     if show_suffix:
                         return _('${ago} ago', mapping={'ago': _val})
                     else:
                         return _(_val)
 
             val = fmt_funcs[part](value)
             val_detail = fmt_funcs[sub_part](sub_value)
             mapping = {'val': val, 'detail': val_detail}
 
             if short_format:
                 datetime_tmpl = _('${val}, ${detail}', mapping=mapping)
                 if show_suffix:
                     datetime_tmpl = _('${val}, ${detail} ago', mapping=mapping)
                     if future:
                         datetime_tmpl = _('in ${val}, ${detail}', mapping=mapping)
             else:
                 datetime_tmpl = _('${val} and ${detail}', mapping=mapping)
                 if show_suffix:
                     datetime_tmpl = _('${val} and ${detail} ago', mapping=mapping)
                     if future:
                         datetime_tmpl = _('in ${val} and ${detail}', mapping=mapping)
 
             return datetime_tmpl
         i += 1
     return _('just now')
 
 
 def age_from_seconds(seconds):
     seconds = safe_int(seconds) or 0
     prevdate = time_to_datetime(time.time() + seconds)
     return age(prevdate, show_suffix=False, show_short_version=True)
 
 
 def cleaned_uri(uri):
     """
     Quotes '[' and ']' from uri if there is only one of them.
         according to RFC3986 we cannot use such chars in uri
     :param uri:
     :return: uri without this chars
     """
     return urllib.parse.quote(uri, safe='@$:/')
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
 
     :param uri:
     """
     import urlobject
     if isinstance(uri, rhodecode.lib.encrypt.InvalidDecryptedValue):
         return 'InvalidDecryptionKey'
 
     url_obj = urlobject.URLObject(cleaned_uri(uri))
     url_obj = url_obj.without_password().without_username()
 
     return url_obj
 
 
 def get_host_info(request):
     """
     Generate host info, to obtain full url e.g https://server.com
     use this
     `{scheme}://{netloc}`
     """
     if not request:
         return {}
 
     qualified_home_url = request.route_url('home')
     parsed_url = urlobject.URLObject(qualified_home_url)
     decoded_path = safe_str(urllib.parse.unquote(parsed_url.path.rstrip('/')))
 
     return {
         'scheme': parsed_url.scheme,
         'netloc': parsed_url.netloc+decoded_path,
         'hostname': parsed_url.hostname,
     }
 
 
 def get_clone_url(request, uri_tmpl, repo_name, repo_id, repo_type, **override):
     qualified_home_url = request.route_url('home')
     parsed_url = urlobject.URLObject(qualified_home_url)
     decoded_path = safe_str(urllib.parse.unquote(parsed_url.path.rstrip('/')))
 
     args = {
         'scheme': parsed_url.scheme,
         'user': '',
         'sys_user': getpass.getuser(),
         # path if we use proxy-prefix
         'netloc': parsed_url.netloc+decoded_path,
         'hostname': parsed_url.hostname,
         'prefix': decoded_path,
         'repo': repo_name,
         'repoid': str(repo_id),
         'repo_type': repo_type
     }
     args.update(override)
     args['user'] = urllib.parse.quote(safe_str(args['user']))
 
     for k, v in list(args.items()):
         tmpl_key = '{%s}' % k
         uri_tmpl = uri_tmpl.replace(tmpl_key, v)
 
     # special case for SVN clone url
     if repo_type == 'svn':
         uri_tmpl = uri_tmpl.replace('ssh://', 'svn+ssh://')
 
     # remove leading @ sign if it's present. Case of empty user
     url_obj = urlobject.URLObject(uri_tmpl)
     url = url_obj.with_netloc(url_obj.netloc.lstrip('@'))
 
     return safe_str(url)
 
 
 def get_commit_safe(repo, commit_id=None, commit_idx=None, pre_load=None,
                     maybe_unreachable=False, reference_obj=None):
     """
     Safe version of get_commit if this commit doesn't exists for a
     repository it returns a Dummy one instead
 
     :param repo: repository instance
     :param commit_id: commit id as str
     :param commit_idx: numeric commit index
     :param pre_load: optional list of commit attributes to load
     :param maybe_unreachable: translate unreachable commits on git repos
     :param reference_obj: explicitly search via a reference obj in git. E.g "branch:123" would mean branch "123"
     """
     # TODO(skreft): remove these circular imports
     from rhodecode.lib.vcs.backends.base import BaseRepository, EmptyCommit
     from rhodecode.lib.vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
 
     try:
         commit = repo.get_commit(
             commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load,
             maybe_unreachable=maybe_unreachable, reference_obj=reference_obj)
     except (RepositoryError, LookupError):
         commit = EmptyCommit()
     return commit
 
 
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 
 
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 
 
 def time_to_utcdatetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.utcfromtimestamp(tm)
 
 
 MENTIONS_REGEX = re.compile(
     # ^@ or @ without any special chars in front
     r'(?:^@|[^a-zA-Z0-9\-\_\.]@)'
     # main body starts with letter, then can be . - _
     r'([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)',
     re.VERBOSE | re.MULTILINE)
 
 
 def extract_mentioned_users(s):
     """
     Returns unique usernames from given string s that have @mention
 
     :param s: string to get mentions
     """
     usrs = set()
     for username in MENTIONS_REGEX.findall(s):
         usrs.add(username)
 
     return sorted(list(usrs), key=lambda k: k.lower())
 
 
 def fix_PATH(os_=None):
     """
     Get current active python path, and append it to PATH variable to fix
     issues of subprocess calls and different python versions
     """
     if os_ is None:
         import os
     else:
         os = os_
 
     cur_path = os.path.split(sys.executable)[0]
     os_path = os.environ['PATH']
     if not os.environ['PATH'].startswith(cur_path):
         os.environ['PATH'] = f'{cur_path}:{os_path}'
 
 
 def obfuscate_url_pw(engine):
     _url = engine or ''
     try:
         _url = sqlalchemy.engine.url.make_url(engine)
     except Exception:
         pass
     return repr(_url)
 
 
 def get_server_url(environ):
     req = webob.Request(environ)
     return req.host_url + req.script_name
 
 
 def unique_id(hexlen=32):
     alphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz"
     return suuid(truncate_to=hexlen, alphabet=alphabet)
 
 
 def suuid(url=None, truncate_to=22, alphabet=None):
     """
     Generate and return a short URL safe UUID.
 
     If the url parameter is provided, set the namespace to the provided
     URL and generate a UUID.
 
     :param url to get the uuid for
     :truncate_to: truncate the basic 22 UUID to shorter version
 
     The IDs won't be universally unique any longer, but the probability of
     a collision will still be very low.
     """
     # Define our alphabet.
     _ALPHABET = alphabet or "23456789ABCDEFGHJKLMNPQRSTUVWXYZ"
 
     # If no URL is given, generate a random UUID.
     if url is None:
         unique_id = uuid.uuid4().int
     else:
         unique_id = uuid.uuid3(uuid.NAMESPACE_URL, url).int
 
     alphabet_length = len(_ALPHABET)
     output = []
     while unique_id > 0:
         digit = unique_id % alphabet_length
         output.append(_ALPHABET[digit])
         unique_id = int(unique_id / alphabet_length)
     return "".join(output)[:truncate_to]
 
 
 def get_current_rhodecode_user(request=None):
     """
     Gets rhodecode user from request
     """
     import pyramid.threadlocal
     pyramid_request = request or pyramid.threadlocal.get_current_request()
 
     # web case
     if pyramid_request and hasattr(pyramid_request, 'user'):
         return pyramid_request.user
 
     # api case
     if pyramid_request and hasattr(pyramid_request, 'rpc_user'):
         return pyramid_request.rpc_user
 
     return None
 
 
 def action_logger_generic(action, namespace=''):
     """
     A generic logger for actions useful to the system overview, tries to find
     an acting user for the context of the call otherwise reports unknown user
 
     :param action: logging message eg 'comment 5 deleted'
     :param type: string
 
     :param namespace: namespace of the logging message eg. 'repo.comments'
     :param type: string
 
     """
 
     logger_name = 'rhodecode.actions'
 
     if namespace:
         logger_name += '.' + namespace
 
     log = logging.getLogger(logger_name)
 
     # get a user if we can
     user = get_current_rhodecode_user()
 
     logfunc = log.info
 
     if not user:
         user = '<unknown user>'
         logfunc = log.warning
 
     logfunc(f'Logging action by {user}: {action}')
 
 
 def escape_split(text, sep=',', maxsplit=-1):
     r"""
     Allows for escaping of the separator: e.g. arg='foo\, bar'
 
     It should be noted that the way bash et. al. do command line parsing, those
     single quotes are required.
     """
     escaped_sep = r'\%s' % sep
 
     if escaped_sep not in text:
         return text.split(sep, maxsplit)
 
     before, _mid, after = text.partition(escaped_sep)
     startlist = before.split(sep, maxsplit)  # a regular split is fine here
     unfinished = startlist[-1]
     startlist = startlist[:-1]
 
     # recurse because there may be more escaped separators
     endlist = escape_split(after, sep, maxsplit)
 
     # finish building the escaped value. we use endlist[0] becaue the first
     # part of the string sent in recursion is the rest of the escaped value.
     unfinished += sep + endlist[0]
 
     return startlist + [unfinished] + endlist[1:]  # put together all the parts
 
 
 class OptionalAttr(object):
     """
     Special Optional Option that defines other attribute. Example::
 
         def test(apiuser, userid=Optional(OAttr('apiuser')):
             user = Optional.extract(userid)
             # calls
 
     """
 
     def __init__(self, attr_name):
         self.attr_name = attr_name
 
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
 
     def __call__(self):
         return self
 
 
 # alias
 OAttr = OptionalAttr
 
 
 class Optional(object):
     """
     Defines an optional parameter::
 
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
 
     is equivalent of::
 
         param = Optional.extract(param)
 
     """
 
     def __init__(self, type_):
         self.type_ = type_
 
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
 
     def __call__(self):
         return self.getval()
 
     def getval(self):
         """
         returns value from this Optional instance
         """
         if isinstance(self.type_, OAttr):
             # use params name
             return self.type_.attr_name
         return self.type_
 
     @classmethod
     def extract(cls, val):
         """
         Extracts value from Optional() instance
 
         :param val:
         :return: original value if it's not Optional instance else
             value of instance
         """
         if isinstance(val, cls):
             return val.getval()
         return val
 
 
 def glob2re(pat):
     import fnmatch
     return fnmatch.translate(pat)
 
 
 def parse_byte_string(size_str):
     match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE)
     if not match:
         raise ValueError(f'Given size:{size_str} is invalid, please make sure '
                          f'to use format of <num>(MB|KB)')
 
     _parts = match.groups()
     num, type_ = _parts
     return int(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()]
 
 
 class CachedProperty(object):
     """
     Lazy Attributes. With option to invalidate the cache by running a method
 
     >>> class Foo(object):
     ...
     ...    @CachedProperty
     ...    def heavy_func(self):
     ...        return 'super-calculation'
     ...
     ... foo = Foo()
     ... foo.heavy_func() # first computation
     ... foo.heavy_func() # fetch from cache
     ... foo._invalidate_prop_cache('heavy_func')
 
     # at this point calling foo.heavy_func() will be re-computed
     """
 
     def __init__(self, func, func_name=None):
 
         if func_name is None:
             func_name = func.__name__
         self.data = (func, func_name)
         functools.update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:
             return self
 
         func, func_name = self.data
         value = func(inst)
         inst.__dict__[func_name] = value
         if '_invalidate_prop_cache' not in inst.__dict__:
             inst.__dict__['_invalidate_prop_cache'] = functools.partial(
                 self._invalidate_prop_cache, inst)
         return value
 
     def _invalidate_prop_cache(self, inst, name):
         inst.__dict__.pop(name, None)
 
 
 def retry(func=None, exception=Exception, n_tries=5, delay=5, backoff=1, logger=True):
     """
     Retry decorator with exponential backoff.
 
     Parameters
     ----------
     func : typing.Callable, optional
         Callable on which the decorator is applied, by default None
     exception : Exception or tuple of Exceptions, optional
         Exception(s) that invoke retry, by default Exception
     n_tries : int, optional
         Number of tries before giving up, by default 5
     delay : int, optional
         Initial delay between retries in seconds, by default 5
     backoff : int, optional
         Backoff multiplier e.g. value of 2 will double the delay, by default 1
     logger : bool, optional
         Option to log or print, by default False
 
     Returns
     -------
     typing.Callable
         Decorated callable that calls itself when exception(s) occur.
 
     Examples
     --------
     >>> import random
     >>> @retry(exception=Exception, n_tries=3)
     ... def test_random(text):
     ...    x = random.random()
     ...    if x < 0.5:
     ...        raise Exception("Fail")
     ...    else:
     ...        print("Success: ", text)
     >>> test_random("It works!")
     """
 
     if func is None:
         return functools.partial(
             retry,
             exception=exception,
             n_tries=n_tries,
             delay=delay,
             backoff=backoff,
             logger=logger,
         )
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         _n_tries, n_delay = n_tries, delay
         log = logging.getLogger('rhodecode.retry')
 
         while _n_tries > 1:
             try:
                 return func(*args, **kwargs)
             except exception as e:
                 e_details = repr(e)
                 msg = "Exception on calling func {func}: {e}, " \
                       "Retrying in {n_delay} seconds..."\
                     .format(func=func, e=e_details, n_delay=n_delay)
                 if logger:
                     log.warning(msg)
                 else:
                     print(msg)
                 time.sleep(n_delay)
                 _n_tries -= 1
                 n_delay *= backoff
 
         return func(*args, **kwargs)
 
     return wrapper
 
 
 def user_agent_normalizer(user_agent_raw, safe=True):
     log = logging.getLogger('rhodecode.user_agent_normalizer')
     ua = (user_agent_raw or '').strip().lower()
     ua = ua.replace('"', '')
 
     try:
         if 'mercurial/proto-1.0' in ua:
             ua = ua.replace('mercurial/proto-1.0', '')
             ua = ua.replace('(', '').replace(')', '').strip()
             ua = ua.replace('mercurial ', 'mercurial/')
         elif ua.startswith('git'):
             parts = ua.split(' ')
             if parts:
                 ua = parts[0]
             ua = re.sub(r'\.windows\.\d', '', ua).strip()
 
         return ua
     except Exception:
         log.exception('Failed to parse scm user-agent')
         if not safe:
             raise
 
     return ua
 
 
 def get_available_port(min_port=40000, max_port=55555, use_range=False):
     hostname = ''
     for _check_port in range(min_port, max_port):
         pick_port = 0
         if use_range:
             pick_port = random.randint(min_port, max_port)
 
         with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
             try:
                 s.bind((hostname, pick_port))
                 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                 return s.getsockname()[1]
             except socket.error as e:
                 if e.args[0] in [errno.EADDRINUSE, errno.ECONNREFUSED]:
                     continue
                 raise
             except OSError:
                 continue

rhodecode/lib/vcs/backends/git/commit.py

@@ -1,453 +1,450
 # Copyright (C) 2014-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT commit module
 """
 
 import io
 import configparser
 import logging
 from itertools import chain
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
 from rhodecode.lib.vcs.nodes import (
     FileNode,
     DirNode,
     NodeKind,
     RootNode,
     SubModuleNode,
     LargeFileNode,
 )
 from rhodecode.lib.vcs_common import FILEMODE_LINK
 
 log = logging.getLogger(__name__)
 
 
 class GitCommit(base.BaseCommit):
     """
     Represents state of the repository at single commit id.
     """
 
     _filter_pre_load = [
         # done through a more complex tree walk on parents
         "affected_files",
         # done through subprocess not remote call
         "children",
         # done through a more complex tree walk on parents
         "status",
         # mercurial specific property not supported here
         "obsolete",
         # mercurial specific property not supported here
         "phase",
         # mercurial specific property not supported here
         "hidden",
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         self.repository = repository
         self._remote = repository._remote
         # TODO: johbo: Tweak of raw_id should not be necessary
         self.raw_id = safe_str(raw_id)
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self.nodes = {}
         self._path_mode_cache = {}  # path stats cache, e.g filemode etc
         self._path_type_cache = {}  # path type dir/file/link etc cache
 
         self._submodules = None
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
         for attr, value in result.items():
             if attr in ["author", "message"]:
                 if value:
                     value = safe_str(value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr == "parents":
                 value = self._make_commits(value)
             elif attr == "branch":
                 value = self._set_branch(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def _commit(self):
         return self._remote[self.raw_id]
 
     @LazyProperty
     def _tree_id(self):
         return self._remote[self._commit["tree"]]["id"]
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def message(self):
         return safe_str(self._remote.message(self.id))
 
     @LazyProperty
     def committer(self):
         return safe_str(self._remote.author(self.id))
 
     @LazyProperty
     def author(self):
         return safe_str(self._remote.author(self.id))
 
     @LazyProperty
     def date(self):
         unix_ts, tz = self._remote.date(self.raw_id)
         return utcdate_fromtimestamp(unix_ts, tz)
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         added, modified, deleted = self._changes_cache
         return list(modified), list(modified), list(deleted)
 
     @LazyProperty
     def tags(self):
         tags = [safe_str(name) for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def commit_branches(self):
         branches = []
         for name, commit_id in self.repository.branches.items():
             if commit_id == self.raw_id:
                 branches.append(name)
         return branches
 
     def _set_branch(self, branches):
         if branches:
             # actually commit can have multiple branches in git
             return safe_str(branches[0])
 
     @LazyProperty
     def branch(self):
         branches = self._remote.branch(self.raw_id)
         return self._set_branch(branches)
 
     def _get_path_tree_id_and_type(self, path: bytes):
 
         if path in self._path_type_cache:
             return self._path_type_cache[path]
 
         if path == b"":
             self._path_type_cache[b""] = [self._tree_id, NodeKind.DIR]
             return self._path_type_cache[path]
 
         tree_id, tree_type, tree_mode = self._remote.tree_and_type_for_path(self.raw_id, path)
         if tree_id is None:
             raise self.no_node_at_path(path)
 
         self._path_type_cache[path] = [tree_id, tree_type]
         self._path_mode_cache[path] = tree_mode
 
         return self._path_type_cache[path]
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         _, path_type = self._get_path_tree_id_and_type(path)
         return path_type
 
     def _assert_is_path(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}")
         return path
 
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parent_ids = self._remote.parents(self.id)
         return self._make_commits(parent_ids)
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
 
         children = self._remote.children(self.raw_id)
         return self._make_commits(children)
 
     def _make_commits(self, commit_ids):
         def commit_maker(_commit_id):
             return self.repository.get_commit(commit_id=_commit_id)
 
         return [commit_maker(commit_id) for commit_id in commit_ids]
 
     def get_file_mode(self, path: bytes):
         """
         Returns stat mode of the file at the given `path`.
         """
         path = self._assert_is_path(path)
 
         # ensure path is traversed
         self._get_path_tree_id_and_type(path)
 
         return self._path_mode_cache[path]
 
     def is_link(self, path: bytes):
         path = self._assert_is_path(path)
         if path not in self._path_mode_cache:
             self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
 
         return self._path_mode_cache[path] == FILEMODE_LINK
 
     def is_node_binary(self, path):
         tree_id, _ = self._get_path_tree_id_and_type(path)
         return self._remote.is_binary(tree_id)
 
     def node_md5_hash(self, path):
         path = self._assert_is_path(path)
         return self._remote.md5_hash(self.raw_id, path)
 
     def get_file_content(self, path):
         """
         Returns content of the file at given `path`.
         """
         tree_id, _ = self._get_path_tree_id_and_type(path)
         return self._remote.blob_as_pretty_string(tree_id)
 
     def get_file_content_streamed(self, path):
         tree_id, _ = self._get_path_tree_id_and_type(path)
         stream_method = getattr(self._remote, "stream:blob_as_pretty_string")
         return stream_method(tree_id)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given `path`.
         """
         tree_id, _ = self._get_path_tree_id_and_type(path)
         return self._remote.blob_raw_length(tree_id)
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `GitCommit` objects for
         which file at given `path` has been modified.
         """
         path = self._assert_is_path(path)
         history = self._remote.node_history(self.raw_id, path, limit)
         return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
         """
 
         result = self._remote.node_annotate(self.raw_id, path)
 
         for ln_no, commit_id, content in result:
             yield (
                 ln_no,
                 commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
                 content,
             )
 
     def get_nodes(self, path: bytes, pre_load=None):
 
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'")
         path = self._fix_path(path)
 
-        # call and check tree_id for this path
-        tree_id, _ = self._get_path_tree_id_and_type(path)
-
         path_nodes = []
 
-        for bytes_name, stat_, tree_item_id, node_kind in self._remote.tree_items(tree_id):
+        for obj_name, stat_, tree_item_id, node_kind, pre_load_data in self._remote.get_nodes(self.raw_id, path, pre_load):
             if node_kind is None:
                 raise CommitError(f"Requested object type={node_kind} cannot be determined")
 
-            if path != b"":
-                obj_path = b"/".join((path, bytes_name))
+            if path == b"":
+                obj_path = obj_name
             else:
-                obj_path = bytes_name
+                obj_path = b"/".join((path, obj_name))
 
             # cache file mode for git, since we have it already
             if obj_path not in self._path_mode_cache:
                 self._path_mode_cache[obj_path] = stat_
 
             # cache type
             if node_kind not in self._path_type_cache:
                 self._path_type_cache[obj_path] = [tree_item_id, node_kind]
 
             entry = None
             if obj_path in self.nodes:
                 entry = self.nodes[obj_path]
             else:
                 if node_kind == NodeKind.SUBMODULE:
-                    url = self._get_submodule_url(b"/".join((path, bytes_name)))
-                    entry= SubModuleNode(bytes_name, url=url, commit=tree_item_id, alias=self.repository.alias)
+                    url = self._get_submodule_url(obj_path)
+                    entry= SubModuleNode(obj_name, url=url, commit=tree_item_id, alias=self.repository.alias)
                 elif node_kind == NodeKind.DIR:
                     entry = DirNode(safe_bytes(obj_path), commit=self)
                 elif node_kind == NodeKind.FILE:
-                    entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
+                    entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load_data=pre_load_data)
 
             if entry:
                 self.nodes[obj_path] = entry
                 path_nodes.append(entry)
 
         path_nodes.sort()
         return path_nodes
 
     def get_node(self, path: bytes, pre_load=None):
         path = self._fix_path(path)
 
         # use cached, if we have one
         if path in self.nodes:
             return self.nodes[path]
 
         try:
             tree_id, path_type = self._get_path_tree_id_and_type(path)
         except CommitError:
             raise NodeDoesNotExistError(f"Cannot find one of parents' directories for a given path: {path}")
 
         if path == b"":
             node = RootNode(commit=self)
         else:
             if path_type == NodeKind.SUBMODULE:
                 url = self._get_submodule_url(path)
                 node = SubModuleNode(path, url=url, commit=tree_id, alias=self.repository.alias)
             elif path_type == NodeKind.DIR:
                 node = DirNode(safe_bytes(path), commit=self)
             elif path_type == NodeKind.FILE:
                 node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
                 self._path_mode_cache[path] = node.mode
             else:
                 raise self.no_node_at_path(path)
 
         # cache node
         self.nodes[path] = node
         return self.nodes[path]
 
     def get_largefile_node(self, path: bytes):
         tree_id, _ = self._get_path_tree_id_and_type(path)
         pointer_spec = self._remote.is_large_file(tree_id)
 
         if pointer_spec:
             # content of that file regular FileNode is the hash of largefile
             file_id = pointer_spec.get("oid_hash")
             if not self._remote.in_largefiles_store(file_id):
                 log.warning(f'Largefile oid={file_id} not found in store')
                 return None
 
             lf_path = self._remote.store_path(file_id)
             return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
 
     @LazyProperty
     def affected_files(self) -> list[bytes]:
         """
         Gets a fast accessible file changes for given commit
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
 
     @LazyProperty
     def _changes_cache(self) -> tuple[set, set, set]:
         added = set()
         modified = set()
         deleted = set()
 
         parents = self.parents
         if not self.parents:
             parents = [base.EmptyCommit()]
         for parent in parents:
             if isinstance(parent, base.EmptyCommit):
                 oid = None
             else:
                 oid = parent.raw_id
             _added, _modified, _deleted = self._remote.tree_changes(oid, self.raw_id)
             added = added | set(_added)
             modified = modified | set(_modified)
             deleted = deleted | set(_deleted)
 
         return added, modified, deleted
 
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
 
         :param status: one of: *added*, *modified* or *deleted*
         """
         added, modified, deleted = self._changes_cache
         return sorted({"added": list(added), "modified": list(modified), "deleted": list(deleted)}[status])
 
     @LazyProperty
     def added_paths(self):
         return [n for n in self._get_paths_for_status("added")]
 
     @LazyProperty
     def changed_paths(self):
         return [n for n in self._get_paths_for_status("modified")]
 
     @LazyProperty
     def removed_paths(self):
         return [n for n in self._get_paths_for_status("deleted")]
 
     def _get_submodule_url(self, submodule_path: bytes):
         git_modules_path = b".gitmodules"
 
         if self._submodules is None:
             self._submodules = {}
 
             try:
                 submodules_node = self.get_node(git_modules_path)
             except NodeDoesNotExistError:
                 return None
 
             parser = configparser.RawConfigParser()
             parser.read_file(io.StringIO(submodules_node.str_content))
 
             for section in parser.sections():
                 path = parser.get(section, "path")
                 url = parser.get(section, "url")
                 if path and url:
                     self._submodules[safe_bytes(path).strip(b"/")] = url
 
         return self._submodules.get(submodule_path.strip(b"/"))

rhodecode/lib/vcs/backends/hg/commit.py

@@ -1,397 +1,397
 # Copyright (C) 2014-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 HG commit module
 """
 import os
 import logging
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError
 from rhodecode.lib.vcs.nodes import (
     DirNode,
     FileNode,
     NodeKind,
     RootNode,
     SubModuleNode,
     LargeFileNode,
 )
 from rhodecode.lib.vcs_common import FILEMODE_LINK
 
 log = logging.getLogger(__name__)
 
 
 class MercurialCommit(base.BaseCommit):
     """
     Represents state of the repository at the single commit.
     """
 
     _filter_pre_load = [
         # git specific property not supported here
         "_commit",
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         raw_id = safe_str(raw_id)
 
         self.repository = repository
         self._remote = repository._remote
 
         self.raw_id = raw_id
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self.nodes = {}
         self._path_mode_cache = {}  # path stats cache, e.g filemode etc
         self._path_type_cache = {}  # path type dir/file/link etc cache
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
 
         for attr, value in result.items():
             if attr in ["author", "branch", "message"]:
                 value = safe_str(value)
             elif attr == "affected_files":
                 value = list(map(safe_str, value))
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr in ["children", "parents"]:
                 value = self._make_commits(value)
             elif attr in ["phase"]:
                 value = self._get_phase_text(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def tags(self):
         tags = [name for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def branch(self):
         return safe_str(self._remote.ctx_branch(self.raw_id))
 
     @LazyProperty
     def bookmarks(self):
         bookmarks = [name for name, commit_id in self.repository.bookmarks.items() if commit_id == self.raw_id]
         return bookmarks
 
     @LazyProperty
     def message(self):
         return safe_str(self._remote.ctx_description(self.raw_id))
 
     @LazyProperty
     def committer(self):
         return safe_str(self.author)
 
     @LazyProperty
     def author(self):
         return safe_str(self._remote.ctx_user(self.raw_id))
 
     @LazyProperty
     def date(self):
         return utcdate_fromtimestamp(*self._remote.ctx_date(self.raw_id))
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         modified, added, deleted, *_ = self._remote.ctx_status(self.raw_id)
         return modified, added, deleted
 
     @LazyProperty
     def id(self):
         if self.last:
             return "tip"
         return self.short_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def _make_commits(self, commit_ids, pre_load=None):
         return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in commit_ids]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parents = self._remote.ctx_parents(self.raw_id)
         return self._make_commits(parents)
 
     def _get_phase_text(self, phase_id):
         return {
             0: "public",
             1: "draft",
             2: "secret",
         }.get(phase_id) or ""
 
     @LazyProperty
     def phase(self):
         phase_id = self._remote.ctx_phase(self.raw_id)
         phase_text = self._get_phase_text(phase_id)
 
         return safe_str(phase_text)
 
     @LazyProperty
     def obsolete(self):
         obsolete = self._remote.ctx_obsolete(self.raw_id)
         return obsolete
 
     @LazyProperty
     def hidden(self):
         hidden = self._remote.ctx_hidden(self.raw_id)
         return hidden
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         children = self._remote.ctx_children(self.raw_id)
         return self._make_commits(children)
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         path_type = self._get_path_type(path)
         return path_type
 
     def _assert_is_path(self, path) -> str | bytes:
         path = self._fix_path(path)
 
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}")
 
         return path
 
     def get_file_mode(self, path: bytes):
         """
         Returns stat mode of the file at the given ``path``.
         """
         path = self._assert_is_path(path)
         if path not in self._path_mode_cache:
             self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
 
         return self._path_mode_cache[path]
 
     def is_link(self, path: bytes):
         path = self._assert_is_path(path)
         if path not in self._path_mode_cache:
             self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
 
         return self._path_mode_cache[path] == FILEMODE_LINK
 
     def is_node_binary(self, path):
         path = self._assert_is_path(path)
         return self._remote.is_binary(self.raw_id, path)
 
     def node_md5_hash(self, path):
         path = self._assert_is_path(path)
         return self._remote.md5_hash(self.raw_id, path)
 
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         path = self._assert_is_path(path)
         return self._remote.fctx_node_data(self.raw_id, path)
 
     def get_file_content_streamed(self, path):
         path = self._assert_is_path(path)
         stream_method = getattr(self._remote, "stream:fctx_node_data")
         return stream_method(self.raw_id, path)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         path = self._assert_is_path(path)
         return self._remote.fctx_size(self.raw_id, path)
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `MercurialCommit` objects
         for which file at given ``path`` has been modified.
         """
         path = self._assert_is_path(path)
         history = self._remote.node_history(self.raw_id, path, limit)
         return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
         """
         result = self._remote.fctx_annotate(self.raw_id, path)
 
         for ln_no, commit_id, content in result:
             yield (
                 ln_no,
                 commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
                 content,
             )
 
     def get_nodes(self, path: bytes, pre_load=None):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of commit at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, CommitError would be raised.
         """
 
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(f"Directory does not exist for idx {self.raw_id} at '{path}'")
         path = self._fix_path(path)
 
         path_nodes = []
 
-        for obj_path, (node_kind, flags) in self._remote.dir_items(self.raw_id, path):
+        for obj_path, node_kind, flags, pre_load_data in self._remote.get_nodes(self.raw_id, path, pre_load):
 
             if node_kind is None:
                 raise CommitError(f"Requested object type={node_kind} cannot be mapped to a proper type")
 
             stat_ = flags
             # cache file mode
             if obj_path not in self._path_mode_cache:
                 self._path_mode_cache[obj_path] = stat_
 
             # cache type
             if node_kind not in self._path_type_cache:
                 self._path_type_cache[obj_path] = node_kind
 
             entry = None
             if obj_path in self.nodes:
                 entry = self.nodes[obj_path]
             else:
                 if node_kind == NodeKind.DIR:
                     entry = DirNode(safe_bytes(obj_path), commit=self)
                 elif node_kind == NodeKind.FILE:
-                    entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
+                    entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load, pre_load_data=pre_load_data)
             if entry:
                 self.nodes[obj_path] = entry
                 path_nodes.append(entry)
 
         for obj_path, (location, commit, scm_type) in self._submodules.items():
 
             if os.path.dirname(obj_path) == path:
                 entry = SubModuleNode(obj_path, url=location, commit=commit, alias=scm_type)
                 self.nodes[obj_path] = entry
                 path_nodes.append(entry)
 
         path_nodes.sort()
         return path_nodes
 
     def get_node(self, path: bytes, pre_load=None):
         """
         Returns `Node` object from the given `path`. If there is no node at
         the given `path`, `NodeDoesNotExistError` would be raised.
         """
         path = self._fix_path(path)
 
         # use cached, if we have one
         if path in self.nodes:
             return self.nodes[path]
 
         path_type = self._get_path_type(path)
         if path == b"":
             node = RootNode(commit=self)
         else:
             if path_type == NodeKind.DIR:
                 node = DirNode(safe_bytes(path), commit=self)
             elif path_type == NodeKind.FILE:
                 node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
                 self._path_mode_cache[path] = node.mode
             else:
                 raise self.no_node_at_path(path)
         # cache node
         self.nodes[path] = node
         return self.nodes[path]
 
     def _get_path_type(self, path: bytes):
         if path in self._path_type_cache:
             return self._path_type_cache[path]
 
         if path == b"":
             self._path_type_cache[b""] = NodeKind.DIR
             return NodeKind.DIR
 
         path_type, flags = self._remote.get_path_type(self.raw_id, path)
 
         if not path_type:
             raise self.no_node_at_path(path)
 
         self._path_type_cache[path] = path_type
         self._path_mode_cache[path] = flags
 
         return self._path_type_cache[path]
 
     def get_largefile_node(self, path: bytes):
         pointer_spec = self._remote.is_large_file(self.raw_id, path)
         if pointer_spec:
             # content of that file regular FileNode is the hash of largefile
             file_id = self.get_file_content(path).strip()
 
             if self._remote.in_largefiles_store(file_id):
                 lf_path = self._remote.store_path(file_id)
                 return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
             elif self._remote.in_user_cache(file_id):
                 lf_path = self._remote.store_path(file_id)
                 self._remote.link(file_id, path)
                 return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
 
     @LazyProperty
     def _submodules(self):
         """
         Returns a dictionary with submodule information from substate file
         of hg repository.
         """
         return self._remote.ctx_substate(self.raw_id)
 
     @LazyProperty
     def affected_files(self) -> list[bytes]:
         """
         Gets a fast accessible file changes for given commit
         """
         return self._remote.ctx_files(self.raw_id)
 
     @LazyProperty
     def added_paths(self):
         return [n for n in self.status[1]]
 
     @LazyProperty
     def changed_paths(self):
         return [n for n in self.status[0]]
 
 
     @LazyProperty
     def removed_paths(self):
         return [n for n in self.status[2]]

rhodecode/lib/vcs/backends/svn/commit.py

@@ -1,278 +1,278
 # Copyright (C) 2014-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SVN commit module
 """
 import logging
 import dateutil.parser
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs import nodes, path as vcspath
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError
 from vcsserver.lib.vcs_common import NodeKind, FILEMODE_EXECUTABLE, FILEMODE_DEFAULT, FILEMODE_LINK
 _SVN_PROP_TRUE = "*"
 
 log = logging.getLogger(__name__)
 
 
 class SubversionCommit(base.BaseCommit):
     """
     Subversion specific implementation of commits
 
     .. attribute:: branch
 
        The Subversion backend does not support to assign branches to
        specific commits. This attribute has always the value `None`.
 
     """
 
     def __init__(self, repository, commit_id):
         self.repository = repository
         self.idx = self.repository._get_commit_idx(commit_id)
         self._svn_rev = self.idx + 1
         self._remote = repository._remote
         # TODO: handling of raw_id should be a method on repository itself,
         # which knows how to translate commit index and commit id
         self.raw_id = commit_id
         self.short_id = commit_id
         self.id = f"r{commit_id}"
 
         self.nodes = {}
         self._path_mode_cache = {}  # path stats cache, e.g filemode etc
         self._path_type_cache = {}  # path type dir/file/link etc cache
         self.tags = []
 
     @property
     def author(self):
         return safe_str(self._properties.get("svn:author"))
 
     @property
     def date(self):
         return _date_from_svn_properties(self._properties)
 
     @property
     def message(self):
         return safe_str(self._properties.get("svn:log"))
 
     @LazyProperty
     def _properties(self):
         return self._remote.revision_properties(self._svn_rev)
 
     @LazyProperty
     def parents(self):
         parent_idx = self.idx - 1
         if parent_idx >= 0:
             parent = self.repository.get_commit(commit_idx=parent_idx)
             return [parent]
         return []
 
     @LazyProperty
     def children(self):
         child_idx = self.idx + 1
         if child_idx < len(self.repository.commit_ids):
             child = self.repository.get_commit(commit_idx=child_idx)
             return [child]
         return []
 
     def _calculate_file_mode(self, path: bytes):
         # Note: Subversion flags files which are executable with a special
         # property `svn:executable` which is set to the value ``"*"``.
         if self._get_file_property(path, "svn:executable") == _SVN_PROP_TRUE:
             return FILEMODE_EXECUTABLE
         else:
             return FILEMODE_DEFAULT
 
     def get_file_mode(self, path: bytes):
         path = self._fix_path(path)
 
         if path not in self._path_mode_cache:
             self._path_mode_cache[path] = self._calculate_file_mode(path)
 
         return self._path_mode_cache[path]
 
     def _get_path_type(self, path: bytes):
         if path in self._path_type_cache:
             return self._path_type_cache[path]
 
         if path == b"":
             self._path_type_cache[b""] = NodeKind.DIR
             return NodeKind.DIR
 
         path_type = self._remote.get_node_type(self._svn_rev, path)
 
         if not path_type:
             raise self.no_node_at_path(path)
 
         #flags = None
         self._path_type_cache[path] = path_type
         #self._path_mode_cache[path] = flags
 
         return self._path_type_cache[path]
 
     def is_link(self, path: bytes):
         # Note: Subversion has a flag for special files, the content of the
         # file contains the type of that file.
         if self._get_file_property(path, "svn:special") == _SVN_PROP_TRUE:
             return self.get_file_content(path).startswith(b"link")
         return False
 
     def is_node_binary(self, path):
         path = self._fix_path(path)
         return self._remote.is_binary(self._svn_rev, safe_str(path))
 
     def node_md5_hash(self, path):
         path = self._fix_path(path)
         return self._remote.md5_hash(self._svn_rev, safe_str(path))
 
     def _get_file_property(self, path, name):
         file_properties = self._remote.node_properties(safe_str(path), self._svn_rev)
         return file_properties.get(name)
 
     def get_file_content(self, path):
         path = self._fix_path(path)
         return self._remote.get_file_content(self._svn_rev, safe_str(path))
 
     def get_file_content_streamed(self, path):
         path = self._fix_path(path)
 
         stream_method = getattr(self._remote, "stream:get_file_content")
         return stream_method(self._svn_rev, safe_str(path))
 
     def get_file_size(self, path):
         path = self._fix_path(path)
         return self._remote.get_file_size(self._svn_rev, safe_str(path))
 
     def get_path_history(self, path, limit=None, pre_load=None):
         path = self._fix_path(path)
         history = self._remote.node_history(self._svn_rev, safe_str(path), limit)
         return [self.repository.get_commit(commit_id=str(svn_rev)) for svn_rev in history]
 
     def get_file_annotate(self, path, pre_load=None):
         result = self._remote.file_annotate(safe_str(path), self._svn_rev)
 
         for zero_based_line_no, svn_rev, content in result:
             commit_id = str(svn_rev)
             line_no = zero_based_line_no + 1
             yield line_no, commit_id, lambda: self.repository.get_commit(commit_id=commit_id), content
 
     def get_node(self, path: bytes, pre_load=None):
         path = self._fix_path(path)
 
         # use cached, if we have one
         if path in self.nodes:
             return self.nodes[path]
 
         path_type = self._get_path_type(path)
         if path == b"":
             node = nodes.RootNode(commit=self)
         else:
             if path_type == NodeKind.DIR:
                 node = nodes.DirNode(safe_bytes(path), commit=self)
             elif path_type == NodeKind.FILE:
                 node = nodes.FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
                 self._path_mode_cache[path] = node.mode
             else:
                 raise self.no_node_at_path(path)
 
         self.nodes[path] = node
         return self.nodes[path]
 
     def get_nodes(self, path: bytes, pre_load=None):
         if self._get_kind(path) != nodes.NodeKind.DIR:
             raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'")
         path = self._fix_path(path)
 
         path_nodes = []
-        for name, node_kind in self._remote.get_nodes(self._svn_rev, path):
-            obj_path = vcspath.join(path, name)
+
+        for obj_path, node_kind, pre_load_data in self._remote.get_nodes(self._svn_rev, path, pre_load):
 
             if node_kind is None:
                 raise CommitError(f"Requested object type={node_kind} cannot be determined")
 
             # TODO: implement it ??
             stat_ = None
             # # cache file mode
             # if obj_path not in self._path_mode_cache:
             #     self._path_mode_cache[obj_path] = stat_
 
             # cache type
             if node_kind not in self._path_type_cache:
                 self._path_type_cache[obj_path] = node_kind
 
             entry = None
             if obj_path in self.nodes:
                 entry = self.nodes[obj_path]
             else:
                 if node_kind == NodeKind.DIR:
                     entry = nodes.DirNode(safe_bytes(obj_path), commit=self)
                 elif node_kind == NodeKind.FILE:
-                    entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
+                    entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load, pre_load_data=pre_load_data)
             if entry:
                 self.nodes[obj_path] = entry
                 path_nodes.append(entry)
 
         path_nodes.sort()
         return path_nodes
 
     def _get_kind(self, path):
         path = self._fix_path(path)
         path_type = self._get_path_type(path)
         return path_type
 
     @LazyProperty
     def _changes_cache(self):
         return self._remote.revision_changes(self._svn_rev)
 
     @LazyProperty
     def affected_files(self) -> list[bytes]:
         changed_files = set()
         for files in self._changes_cache.values():
             changed_files.update(files)
         return list(changed_files)
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @LazyProperty
     def added_paths(self):
         return [n for n in self._changes_cache["added"]]
 
     @LazyProperty
     def changed_paths(self):
         return [n for n in self._changes_cache["changed"]]
 
     @LazyProperty
     def removed_paths(self):
         return [n for n in self._changes_cache["removed"]]
 
 
 def _date_from_svn_properties(properties):
     """
     Parses the date out of given svn properties.
 
     :return: :class:`datetime.datetime` instance. The object is naive.
     """
 
     aware_date = dateutil.parser.parse(properties.get("svn:date"))
     # final_date = aware_date.astimezone(dateutil.tz.tzlocal())
     final_date = aware_date
     return final_date.replace(tzinfo=None)

rhodecode/lib/vcs/nodes.py

@@ -1,783 +1,788
 # Copyright (C) 2014-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Module holding everything related to vcs nodes, with vcs2 architecture.
 """
 
 import functools
 import os
 import stat
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP
 from rhodecode.lib.str_utils import safe_str, safe_bytes
 from rhodecode.lib.hash_utils import md5
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.lib.vcs.exceptions import NodeError
 from rhodecode.lib.vcs_common import NodeKind, FILEMODE_DEFAULT
 
 LARGEFILE_PREFIX = ".hglf"
 
 
 class NodeState:
     ADDED = "added"
     CHANGED = "changed"
     NOT_CHANGED = "not changed"
     REMOVED = "removed"
 
 
 # TODO: not sure if that should be bytes or str ?
 # most probably bytes because content should be bytes and we check it
 BIN_BYTE_MARKER = b"\0"
 
 
 
 @functools.total_ordering
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
 
     # RTLO marker allows swapping text, and certain
     # security attacks could be used with this
     RTLO_MARKER = "\u202e"
 
     commit = None
 
     def __init__(self, path: bytes, kind):
         self._validate_path(path)  # can throw exception if path is invalid
 
         self.bytes_path: bytes = path.rstrip(b"/")  # store for mixed encoding, and raw version
         self.str_path: str = safe_str(self.bytes_path)  # we store paths as str
         self.path: str = self.str_path
 
         if self.bytes_path == b"" and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be initialized with empty path")
         self.kind = kind
 
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
     def __eq__(self, other):
         if type(self) is not type(other):
             return False
         for attr in ["name", "path", "kind"]:
             if getattr(self, attr) != getattr(other, attr):
                 return False
         if self.is_file():
             # FileNode compare, we need to fallback to content compare
             return None
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             if self_nodes_paths != other_nodes_paths:
                 return False
         return True
 
     def __lt__(self, other):
         if self.kind < other.kind:
             return True
         if self.kind > other.kind:
             return False
         if self.path < other.path:
             return True
         if self.path > other.path:
             return False
 
     def __repr__(self):
         maybe_path = getattr(self, "path", "UNKNOWN_PATH")
         return f"<{self.__class__.__name__} {maybe_path!r}>"
 
     def __str__(self):
         return self.name
 
     def _validate_path(self, path: bytes):
         self._assert_bytes(path)
 
         if path.startswith(b"/"):
             raise NodeError(
                 f"Cannot initialize Node objects with slash at "
                 f"the beginning as only relative paths are supported. "
                 f"Got {path}"
             )
 
     @classmethod
     def _assert_bytes(cls, value):
         if not isinstance(value, bytes):
             raise TypeError(f"Bytes required as input, got {type(value)} of {value}.")
 
     @LazyProperty
     def parent(self):
         parent_path: bytes = self.get_parent_path()
         if parent_path:
             if self.commit:
                 return self.commit.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
     def has_rtlo(self):
         """Detects if a path has right-to-left-override marker"""
         return self.RTLO_MARKER in self.str_path
 
     @LazyProperty
     def dir_path(self):
         """
         Returns name of the directory from full path of this vcs node. Empty
         string is returned if there's no directory in the path
         """
         _parts = self.path.rstrip("/").rsplit("/", 1)
         if len(_parts) == 2:
             return _parts[0]
         return ""
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return self.str_path.rstrip("/").split("/")[-1]
 
     @property
     def kind(self):
         return self._kind
 
     @kind.setter
     def kind(self, kind):
         if hasattr(self, "_kind"):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.str_path.endswith("/"):
                 raise NodeError("Node's path cannot end with slash")
 
     def get_parent_path(self) -> bytes:
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return b""
         str_path = vcspath.dirname(self.bytes_path.rstrip(b"/")) + b"/"
 
         return safe_bytes(str_path)
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ""
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     def is_largefile(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False``
         otherwise
         """
         return self.kind == NodeKind.LARGE_FILE
 
     def is_link(self):
         if self.commit:
             return self.commit.is_link(self.bytes_path)
         return False
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: commit: if given, first time content is accessed, callback
     :attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`.
     """
 
     _filter_pre_load = []
 
-    def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None):
+    def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None, pre_load_data=None):
         """
         Only one of ``content`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         :param mode: ST_MODE (i.e. 0100644)
         """
         if content and commit:
             raise NodeError("Cannot use both content and commit")
 
         super().__init__(path, kind=NodeKind.FILE)
 
         self.commit = commit
         if content and not isinstance(content, bytes):
             # File content is one thing that inherently must be bytes
             # we support passing str too, and convert the content
             content = safe_bytes(content)
         self._content = content
         self._mode = mode or FILEMODE_DEFAULT
-
-        self._set_bulk_properties(pre_load)
+        if pre_load_data:
+            self._store_pre_load(pre_load_data)
+        else:
+            self._set_bulk_properties(pre_load)
 
     def __eq__(self, other):
         eq = super().__eq__(other)
         if eq is not None:
             return eq
         return self.content == other.content
 
     def __hash__(self):
         raw_id = getattr(self.commit, "raw_id", "")
         return hash((self.path, raw_id))
 
     def __lt__(self, other):
         lt = super().__lt__(other)
         if lt is not None:
             return lt
         return self.content < other.content
 
     def __repr__(self):
         short_id = getattr(self.commit, "short_id", "")
         return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         remote = self.commit.get_remote()
         result = remote.bulk_file_request(self.commit.raw_id, self.bytes_path, pre_load)
 
-        for attr, value in result.items():
+        self._store_pre_load(result.items())
+
+    def _store_pre_load(self, pre_load_data):
+        for attr, value in pre_load_data:
             if attr == "flags":
                 self.__dict__["mode"] = safe_str(value)
             elif attr == "size":
                 self.__dict__["size"] = value
             elif attr == "data":
                 self.__dict__["_content"] = value
             elif attr == "is_binary":
                 self.__dict__["is_binary"] = value
             elif attr == "md5":
                 self.__dict__["md5"] = value
             else:
                 raise ValueError(f"Unsupported attr in bulk_property: {attr}")
 
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If `commit` is not set, would
         use value given at initialization or `FILEMODE_DEFAULT` (default).
         """
         if self.commit:
             mode = self.commit.get_file_mode(self.bytes_path)
         else:
             mode = self._mode
         return mode
 
     @LazyProperty
     def raw_bytes(self) -> bytes:
         """
         Returns lazily the raw bytes of the FileNode.
         """
         if self.commit:
             if self._content is None:
                 self._content = self.commit.get_file_content(self.bytes_path)
             content = self._content
         else:
             content = self._content
         return content
 
     def content_uncached(self):
         """
         Returns lazily content of the FileNode.
         """
         if self.commit:
             content = self.commit.get_file_content(self.bytes_path)
         else:
             content = self._content
         return content
 
     def stream_bytes(self):
         """
         Returns an iterator that will stream the content of the file directly from
         vcsserver without loading it to memory.
         """
         if self.commit:
             return self.commit.get_file_content_streamed(self.bytes_path)
         raise NodeError("Cannot retrieve stream_bytes without related commit attribute")
 
     def metadata_uncached(self):
         """
         Returns md5, binary flag of the file node, without any cache usage.
         """
 
         content = self.content_uncached()
 
         is_binary = bool(content and BIN_BYTE_MARKER in content)
         size = 0
         if content:
             size = len(content)
 
         return is_binary, md5(content), size, content
 
     @LazyProperty
     def content(self) -> bytes:
         """
         Returns lazily content of the FileNode.
         """
         content = self.raw_bytes
         if content and not isinstance(content, bytes):
             raise ValueError(f"Content is of type {type(content)} instead of bytes")
         return content
 
     @LazyProperty
     def str_content(self) -> str:
         return safe_str(self.raw_bytes)
 
     @LazyProperty
     def size(self):
         if self.commit:
             return self.commit.get_file_size(self.bytes_path)
         raise NodeError("Cannot retrieve size of the file without related commit attribute")
 
     @LazyProperty
     def message(self):
         if self.commit:
             return self.last_commit.message
         raise NodeError("Cannot retrieve message of the file without related " "commit attribute")
 
     @LazyProperty
     def last_commit(self):
         if self.commit:
             pre_load = ["author", "date", "message", "parents"]
             return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load)
         raise NodeError("Cannot retrieve last commit of the file without related commit attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content. If ``_mimetype``
         attribute is available, it will be returned (backends which store
         mimetypes or can easily recognize them, should set this private
         attribute to indicate that type should *NOT* be calculated).
         """
 
         if hasattr(self, "_mimetype"):
             if isinstance(self._mimetype, (tuple, list)) and len(self._mimetype) == 2:
                 return self._mimetype
             else:
                 raise NodeError("given _mimetype attribute must be an 2 element list or tuple")
 
         db = get_mimetypes_db()
         mtype, encoding = db.guess_type(self.name)
 
         if mtype is None:
             if not self.is_largefile() and self.is_binary:
                 mtype = "application/octet-stream"
                 encoding = None
             else:
                 mtype = "text/plain"
                 encoding = None
 
                 # try with pygments
                 try:
                     from pygments.lexers import get_lexer_for_filename
 
                     mt = get_lexer_for_filename(self.name).mimetypes
                 except Exception:
                     mt = None
 
                 if mt:
                     mtype = mt[0]
 
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split("/")[0]
 
     @classmethod
     def get_lexer(cls, filename, content=None):
         from pygments import lexers
 
         extension = filename.split(".")[-1]
         lexer = None
 
         try:
             lexer = lexers.guess_lexer_for_filename(filename, content, stripnl=False)
         except lexers.ClassNotFound:
             pass
 
         # try our EXTENSION_MAP
         if not lexer:
             try:
                 lexer_class = LANGUAGES_EXTENSIONS_MAP.get(extension)
                 if lexer_class:
                     lexer = lexers.get_lexer_by_name(lexer_class[0])
             except lexers.ClassNotFound:
                 pass
 
         if not lexer:
             lexer = lexers.TextLexer(stripnl=False)
 
         return lexer
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
         # TODO: this is more proper, but super heavy on investigating the type based on the content
         # self.get_lexer(self.name, self.content)
 
         return self.get_lexer(self.name)
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of commit for this file in which the file was changed
         """
         if self.commit is None:
             raise NodeError("Unable to get commit for this FileNode")
         return self.commit.get_path_history(self.bytes_path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno, commit and line
         """
         if self.commit is None:
             raise NodeError("Unable to get commit for this FileNode")
         pre_load = ["author", "date", "message", "parents"]
         return self.commit.get_file_annotate(self.bytes_path, pre_load=pre_load)
 
     @LazyProperty
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
         if self.commit:
             return self.commit.is_node_binary(self.bytes_path)
         else:
             raw_bytes = self._content
             return bool(raw_bytes and BIN_BYTE_MARKER in raw_bytes)
 
     @LazyProperty
     def md5(self):
         """
         Returns md5 of the file node.
         """
 
         if self.commit:
             return self.commit.node_md5_hash(self.bytes_path)
         else:
             raw_bytes = self._content
             # TODO: this sucks, we're computing md5 on potentially super big stream data...
             return md5(raw_bytes)
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split(".")[-1]
 
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def get_largefile_node(self):
         """
         Try to return a Mercurial FileNode from this node. It does internal
         checks inside largefile store, if that file exist there it will
         create special instance of LargeFileNode which can get content from
         LF store.
         """
         if self.commit:
             return self.commit.get_largefile_node(self.bytes_path)
 
     def count_lines(self, content: str | bytes, count_empty=False):
         if isinstance(content, str):
             newline_marker = "\n"
         elif isinstance(content, bytes):
             newline_marker = b"\n"
         else:
             raise ValueError("content must be bytes or str got {type(content)} instead")
 
         if count_empty:
             all_lines = 0
             empty_lines = 0
             for line in content.splitlines(True):
                 if line == newline_marker:
                     empty_lines += 1
                 all_lines += 1
 
             return all_lines, all_lines - empty_lines
         else:
             # fast method
             empty_lines = all_lines = content.count(newline_marker)
             if all_lines == 0 and content:
                 # one-line without a newline
                 empty_lines = all_lines = 1
 
         return all_lines, empty_lines
 
     def lines(self, count_empty=False):
         all_lines, empty_lines = 0, 0
 
         if not self.is_binary:
             content = self.content
             all_lines, empty_lines = self.count_lines(content, count_empty=count_empty)
         return all_lines, empty_lines
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repository's commit.
     """
 
     def __init__(self, path, nodes=(), commit=None, default_pre_load=None):
         """
         Only one of ``nodes`` and ``commit`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param commit: if given, will use it to lazily fetch content
         """
         if nodes and commit:
             raise NodeError("Cannot use both nodes and commit")
         super().__init__(path, NodeKind.DIR)
         self.commit = commit
         self._nodes = nodes
         self.default_pre_load = default_pre_load or ["is_binary", "size"]
 
     def __iter__(self):
         yield from self.nodes
 
     def __eq__(self, other):
         eq = super().__eq__(other)
         if eq is not None:
             return eq
         # check without entering each dir
         self_nodes_paths = list(sorted(n.path for n in self.nodes))
         other_nodes_paths = list(sorted(n.path for n in self.nodes))
         return self_nodes_paths == other_nodes_paths
 
     def __lt__(self, other):
         lt = super().__lt__(other)
         if lt is not None:
             return lt
         # check without entering each dir
         self_nodes_paths = list(sorted(n.path for n in self.nodes))
         other_nodes_paths = list(sorted(n.path for n in self.nodes))
         return self_nodes_paths < other_nodes_paths
 
     @LazyProperty
     def content(self):
         raise NodeError(f"{self} represents a dir and has no `content` attribute")
 
     @LazyProperty
     def nodes(self):
         if self.commit:
             nodes = self.commit.get_nodes(self.bytes_path, pre_load=self.default_pre_load)
         else:
             nodes = self._nodes
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted(node for node in self.nodes if node.is_file())
 
     @LazyProperty
     def dirs(self):
         return sorted(node for node in self.nodes if node.is_dir())
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.commit.walk(self.bytes_path):
             for f in files:
                 size += f.size
 
         return size
 
     @LazyProperty
     def last_commit(self):
         if self.commit:
             pre_load = ["author", "date", "message", "parents"]
             return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load)
         raise NodeError("Cannot retrieve last commit of the file without related commit attribute")
 
     def __repr__(self):
         short_id = getattr(self.commit, "short_id", "")
         return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), commit=None):
         super().__init__(path=b"", nodes=nodes, commit=commit)
 
     def __repr__(self):
         short_id = getattr(self.commit, "short_id", "")
         return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
 
     is_binary = False
     size = 0
 
     def __init__(self, name, url=None, commit=None, alias=None):
         self.path: bytes = name
         self.str_path: str = safe_str(self.path)  # we store paths as str
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
 
         # we have to use EmptyCommit here since this can point to svn/git/hg
         # submodules we cannot get from repository
         self.commit = EmptyCommit(safe_str(commit), alias=alias)
         self.url = safe_str(url) or self._extract_submodule_url()
 
     def __repr__(self):
         short_id = getattr(self.commit, "short_id", "")
         return f"<{self.__class__.__name__} {self.str_path!r} @ {short_id}>"
 
     def _extract_submodule_url(self):
         # TODO: find a way to parse gits submodule file and extract the linking URL
         return safe_str(self.path)
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = self.str_path.rstrip("/").split("/")[-1]
         return f"{org} @ {self.commit.short_id}"
 
 
 class LargeFileNode(FileNode):
     def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
         self._validate_path(path)  # can throw exception if path is invalid
         self.org_path = org_path  # as stored in VCS as LF pointer
 
         self.bytes_path = path.rstrip(b"/")  # store for __repr__
         self.str_path = safe_str(self.bytes_path)
         self.path = self.str_path
 
         self.kind = NodeKind.LARGE_FILE
         self.alias = alias
         self._content = b""
 
     def _validate_path(self, path: bytes):
         """
         we override check since the LargeFileNode path is system absolute, but we check for bytes only
         """
         self._assert_bytes(path)
 
     def __repr__(self):
         return f"<{self.__class__.__name__} {self.org_path} -> {self.str_path!r}>"
 
     @LazyProperty
     def size(self):
         return os.stat(self.path).st_size
 
     @LazyProperty
     def raw_bytes(self):
         with open(self.path, "rb") as f:
             content = f.read()
         return content
 
     @LazyProperty
     def name(self):
         """
         Overwrites name to be the org lf path
         """
         return self.org_path
 
     def stream_bytes(self):
         with open(self.path, "rb") as stream:
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     break
                 yield data

rhodecode/model/repo.py

@@ -1,1218 +1,1220
 # Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import shutil
 import time
 import logging
 import traceback
 import datetime
 
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.orm import aliased
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode import events
 from rhodecode.lib.auth import HasUserGroupPermissionAny
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib import hooks_base
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.utils2 import (
     safe_str, remove_prefix, obfuscate_url_pw,
     get_current_rhodecode_user, safe_int, action_logger_generic)
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.lib.vcs.nodes import NodeKind
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     _hash_key, func, case, joinedload, or_, in_filter_generator,
     Session, Repository, UserRepoToPerm, UserGroupRepoToPerm,
     UserRepoGroupToPerm, UserGroupRepoGroupToPerm, User, Permission,
     Statistics, UserGroup, RepoGroup, RepositoryField, UserLog)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(BaseModel):
 
     cls = Repository
 
     def _get_user_group(self, users_group):
         return self._get_instance(UserGroup, users_group,
                                   callback=UserGroup.get_by_group_name)
 
     def _get_repo_group(self, repo_group):
         return self._get_instance(RepoGroup, repo_group,
                                   callback=RepoGroup.get_by_group_name)
 
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
 
         default_perm = 'repository.none' if private else default
 
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
         repo_to_perm.user = def_user
 
         return repo_to_perm
 
     def get(self, repo_id):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_id == repo_id)
 
         return repo.scalar()
 
     def get_repo(self, repository):
         return self._get_repo(repository)
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             name_key = _hash_key(repo_name)
             repo = repo.options(
                 FromCache("sql_cache_short", f"get_repo_{name_key}"))
         return repo.scalar()
 
     def _extract_id_from_repo_name(self, repo_name):
         if repo_name.startswith('/'):
             repo_name = repo_name.lstrip('/')
         by_id_match = re.match(r'^_(\d+)', repo_name)
         if by_id_match:
             return by_id_match.groups()[0]
 
     def get_repo_by_id(self, repo_name):
         """
         Extracts repo_name by id from special urls.
         Example url is _11/repo_name
 
         :param repo_name:
         :return: repo object if matched else None
         """
         _repo_id = None
         try:
             _repo_id = self._extract_id_from_repo_name(repo_name)
             if _repo_id:
                 return self.get(_repo_id)
         except Exception:
             log.exception('Failed to extract repo_name from URL')
             if _repo_id:
                 Session().rollback()
 
         return None
 
     def get_repos_for_root(self, root, traverse=False):
         if traverse:
             like_expression = u'{}%'.format(safe_str(root))
             repos = Repository.query().filter(
                 Repository.repo_name.like(like_expression)).all()
         else:
             if root and not isinstance(root, RepoGroup):
                 raise ValueError(
                     'Root must be an instance '
                     'of RepoGroup, got:{} instead'.format(type(root)))
             repos = Repository.query().filter(Repository.group == root).all()
         return repos
 
     def get_url(self, repo, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_summary', repo_name='_{}'.format(safe_str(repo.repo_id)))
         else:
             return request.route_url(
                 'repo_summary', repo_name=safe_str(repo.repo_name))
 
     def get_commit_url(self, repo, commit_id, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_id),
                 commit_id=commit_id)
 
         else:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_name),
                 commit_id=commit_id)
 
     def get_repo_log(self, repo, filter_term):
         repo_log = UserLog.query()\
             .filter(or_(UserLog.repository_id == repo.repo_id,
                         UserLog.repository_name == repo.repo_name))\
             .options(joinedload(UserLog.user))\
             .options(joinedload(UserLog.repository))\
             .order_by(UserLog.action_date.desc())
 
         repo_log = user_log_filter(repo_log, filter_term)
         return repo_log
 
     @classmethod
     def update_commit_cache(cls, repositories=None):
         if not repositories:
             repositories = Repository.getAll()
         for repo in repositories:
             repo.update_commit_cache()
 
     def get_repos_as_dict(self, repo_list=None, admin=False,
                           super_user_actions=False, short_name=None):
 
         _render = get_current_request().get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
         c = _render.get_call_context()
         h = _render.get_helpers()
 
         def quick_menu(repo_name):
             return _render('quick_menu', repo_name)
 
         def repo_lnk(name, rtype, rstate, private, archived, fork_repo_name):
             if short_name is not None:
                 short_name_var = short_name
             else:
                 short_name_var = not admin
             return _render('repo_name', name, rtype, rstate, private, archived, fork_repo_name,
                            short_name=short_name_var, admin=False)
 
         def last_change(last_change):
             if admin and isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
                 ts = time.time()
                 utc_offset = (datetime.datetime.fromtimestamp(ts)
                               - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
                 last_change = last_change + datetime.timedelta(seconds=utc_offset)
 
             return _render("last_change", last_change)
 
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
 
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
 
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'), cs_cache.get('date'))
 
         def desc(desc):
             return _render('repo_desc', desc, c.visual.stylify_metatags)
 
         def state(repo_state):
             return _render("repo_state", repo_state)
 
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name, super_user_actions)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         repos_data = []
         for repo in repo_list:
             # NOTE(marcink): because we use only raw column we need to load it like that
             changeset_cache = Repository._load_changeset_cache(
                 repo.repo_id, repo._changeset_cache)
 
             row = {
                 "menu": quick_menu(repo.repo_name),
 
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
                                  repo.private, repo.archived, repo.fork_repo_name),
 
                 "desc": desc(h.escape(repo.description)),
 
                 "last_change": last_change(repo.updated_on),
 
                 "last_changeset": last_rev(repo.repo_name, changeset_cache),
                 "last_changeset_raw": changeset_cache.get('revision'),
 
                 "owner": user_profile(repo.owner_username),
 
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
             }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                 })
             repos_data.append(row)
 
         return repos_data
 
     def get_repos_data_table(
             self, draw, start, limit,
             search_q, order_by, order_dir,
             auth_user, repo_group_id):
         from rhodecode.model.scm import RepoList
 
         _perms = ['repository.read', 'repository.write', 'repository.admin']
 
         repos = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .all()
         auth_repo_list = RepoList(
             repos, perm_set=_perms,
             extra_kwargs=dict(user=auth_user))
 
         allowed_ids = [-1]
         for repo in auth_repo_list:
             allowed_ids.append(repo.repo_id)
 
         repos_data_total_count = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .count()
 
         RepoFork = aliased(Repository)
         OwnerUser = aliased(User)
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
             Repository.description,
             Repository.repo_type,
             Repository.repo_state,
             Repository.private,
             Repository.archived,
             Repository.updated_on,
             Repository._changeset_cache,
             RepoFork.repo_name.label('fork_repo_name'),
             OwnerUser.username.label('owner_username'),
             ) \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \
             .join(OwnerUser, Repository.user_id == OwnerUser.user_id)
 
         repos_data_total_filtered_count = base_q.count()
 
         sort_defined = False
         if order_by == 'repo_name':
             sort_col = func.lower(Repository.repo_name)
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(Repository, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         repos_list = base_q.all()
 
         repos_data = RepoModel().get_repos_as_dict(
             repo_list=repos_list, admin=False)
 
         data = ({
             'draw': draw,
             'data': repos_data,
             'recordsTotal': repos_data_total_count,
             'recordsFiltered': repos_data_total_filtered_count,
         })
         return data
 
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
 
         :param repo_name:
         """
 
         repo_info = Repository.get_by_repo_name(repo_name)
 
         if repo_info is None:
             return None
 
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
 
         groups = repo_info.groups_with_parents
         parent_group = groups[-1] if groups else None
 
         # we use -1 as this is how in HTML, we mark an empty group
         defaults['repo_group'] = getattr(parent_group, 'group_id', -1)
 
         keys_to_process = (
             {'k': 'repo_type', 'strip': False},
             {'k': 'repo_enable_downloads', 'strip': True},
             {'k': 'repo_description', 'strip': True},
             {'k': 'repo_enable_locking', 'strip': True},
             {'k': 'repo_landing_rev', 'strip': True},
             {'k': 'clone_uri', 'strip': False},
             {'k': 'push_uri', 'strip': False},
             {'k': 'repo_private', 'strip': True},
             {'k': 'repo_enable_statistics', 'strip': True}
         )
 
         for item in keys_to_process:
             attr = item['k']
             if item['strip']:
                 attr = remove_prefix(item['k'], 'repo_')
 
             val = defaults[attr]
             if item['k'] == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[item['k']] = val
             if item['k'] == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
             if item['k'] == 'push_uri':
                 defaults['push_uri_hidden'] = repo_info.push_uri_hidden
 
         # fill owner
         if repo_info.user:
             defaults.update({'user': repo_info.user.username})
         else:
             replacement_user = User.get_first_super_admin().username
             defaults.update({'user': replacement_user})
 
         return defaults
 
     def update(self, repo, **kwargs):
         try:
             cur_repo = self._get_repo(repo)
             source_repo_name = cur_repo.repo_name
 
             affected_user_ids = []
             if 'user' in kwargs:
                 old_owner_id = cur_repo.user.user_id
                 new_owner = User.get_by_username(kwargs['user'])
                 cur_repo.user = new_owner
 
                 if old_owner_id != new_owner.user_id:
                     affected_user_ids = [new_owner.user_id, old_owner_id]
 
             if 'repo_group' in kwargs:
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
 
             update_keys = [
                 (1, 'repo_description'),
                 (1, 'repo_landing_rev'),
                 (1, 'repo_private'),
                 (1, 'repo_enable_downloads'),
                 (1, 'repo_enable_locking'),
                 (1, 'repo_enable_statistics'),
                 (0, 'clone_uri'),
                 (0, 'push_uri'),
                 (0, 'fork_id')
             ]
             for strip, k in update_keys:
                 if k in kwargs:
                     val = kwargs[k]
                     if strip:
                         k = remove_prefix(k, 'repo_')
 
                     setattr(cur_repo, k, val)
 
             new_name = source_repo_name
             if 'repo_name' in kwargs:
                 new_name = cur_repo.get_new_name(kwargs['repo_name'])
                 cur_repo.repo_name = new_name
 
             if 'repo_private' in kwargs:
                 # if private flag is set to True, reset default permission to NONE
                 set_private_to = kwargs.get('repo_private')
                 if set_private_to:
                     EMPTY_PERM = 'repository.none'
                     RepoModel().grant_user_permission(
                         repo=cur_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM
                     )
                 if set_private_to != cur_repo.private:
                     # NOTE(dan): we change repo private mode we need to notify all USERS
                     # this is just by having this value set to a different value then it was before
                     affected_user_ids = User.get_all_user_ids()
 
             if kwargs.get('repo_landing_rev'):
                 landing_rev_val = kwargs['repo_landing_rev']
                 RepoModel().set_landing_rev(cur_repo, landing_rev_val)
 
             # handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX), kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(
                     key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
                     self.sa.add(ex_field)
 
             self.sa.add(cur_repo)
 
             if source_repo_name != new_name:
                 # rename repository
                 self._rename_filesystem_repo(
                     old=source_repo_name, new=new_name)
 
             if affected_user_ids:
                 PermissionModel().trigger_permission_flush(affected_user_ids)
 
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev=None, fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_locking=False, enable_downloads=False,
                      copy_group_permissions=False,
                      state=Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state, this should be
         only executed by create() repo. With exception of importing existing
         repos
         """
         from rhodecode.model.scm import ScmModel
 
         owner = self._get_user(owner)
         fork_of = self._get_repo(fork_of)
         repo_group = self._get_repo_group(safe_int(repo_group))
         default_landing_ref, _lbl = ScmModel.backend_landing_ref(repo_type)
         landing_rev = landing_rev or default_landing_ref
 
         try:
             repo_name = safe_str(repo_name)
             description = safe_str(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(Repository.NAME_SEP)[-1]
 
             new_repo = Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.user = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             new_repo.archived = False
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
 
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_locking = enable_locking
             new_repo.enable_downloads = enable_downloads
 
             if repo_group:
                 new_repo.enable_locking = repo_group.enable_locking
 
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
 
             events.trigger(events.RepoPreCreateEvent(new_repo))
 
             self.sa.add(new_repo)
 
             EMPTY_PERM = 'repository.none'
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = UserRepoToPerm.query() \
                     .filter(UserRepoToPerm.repository == repo).all()
                 group_perms = UserGroupRepoToPerm.query() \
                     .filter(UserGroupRepoToPerm.repository == repo).all()
 
                 for perm in user_perms:
                     UserRepoToPerm.create(
                         perm.user, new_repo, perm.permission)
 
                 for perm in group_perms:
                     UserGroupRepoToPerm.create(
                         perm.users_group, new_repo, perm.permission)
                 # in case we copy permissions and also set this repo to private
                 # override the default user permission to make it a private repo
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             elif repo_group and copy_group_permissions:
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == repo_group).all()
 
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == repo_group).all()
 
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserRepoToPerm.create(perm.user, new_repo, perm_obj)
 
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
 
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             else:
                 perm_obj = self._create_default_perms(new_repo, private)
                 self.sa.add(perm_obj)
 
             # now automatically start following this repository as owner
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, owner.user_id)
 
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             self.sa.flush()
             events.trigger(events.RepoCreateEvent(new_repo, actor=owner))
             return new_repo
 
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo, form_data, cur_user)
 
     def update_permissions(self, repo, perm_additions=None, perm_updates=None,
                            perm_deletions=None, check_perms=True,
                            cur_user=None):
         if not perm_additions:
             perm_additions = []
         if not perm_updates:
             perm_updates = []
         if not perm_deletions:
             perm_deletions = []
 
         req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
 
         changes = {
             'added': [],
             'updated': [],
             'deleted': [],
             'default_user_changed': None
         }
 
         repo = self._get_repo(repo)
 
         # update permissions
         for member_id, perm, member_type in perm_updates:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 if member_name == User.DEFAULT_USER:
                     # NOTE(dan): detect if we changed permissions for default user
                     perm_obj = self.sa.query(UserRepoToPerm) \
                         .filter(UserRepoToPerm.user_id == member_id) \
                         .filter(UserRepoToPerm.repository == repo) \
                         .scalar()
                     if perm_obj and perm_obj.permission.permission_name != perm:
                         changes['default_user_changed'] = True
 
                 # this updates also current one if found
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
             changes['updated'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
 
         # set new permissions
         for member_id, perm, member_type in perm_additions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['added'].append({'type': member_type, 'id': member_id,
                                      'name': member_name, 'new_perm': perm})
         # delete permissions
         for member_id, perm, member_type in perm_deletions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.revoke_user_permission(repo=repo, user=member_id)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.revoke_user_group_permission(
                         repo=repo, group_name=member_id)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['deleted'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
         return changes
 
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def archive(self, repo):
         """
         Archive given repository. Set archive flag.
 
         :param repo:
         """
         repo = self._get_repo(repo)
         if repo:
 
             try:
                 repo.archived = True
                 self.sa.add(repo)
                 self.sa.commit()
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None, call_events=True):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
 
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param pull_requests: str 'delete' or None
         :param artifacts: str 'delete' or None
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
         repo = self._get_repo(repo)
         if not repo:
             return False
 
         if forks == 'detach':
             for r in repo.forks:
                 r.fork = None
                 self.sa.add(r)
         elif forks == 'delete':
             for r in repo.forks:
                 self.delete(r, forks='delete')
         elif [f for f in repo.forks]:
             raise AttachedForksError()
 
         # check for pull requests
         pr_sources = repo.pull_requests_source
         pr_targets = repo.pull_requests_target
         if pull_requests != 'delete' and (pr_sources or pr_targets):
             raise AttachedPullRequestsError()
 
         artifacts_objs = repo.artifacts
         if artifacts == 'delete':
             for a in artifacts_objs:
                 self.sa.delete(a)
         elif [a for a in artifacts_objs]:
             raise AttachedArtifactsError()
 
         old_repo_dict = repo.get_dict()
         if call_events:
             events.trigger(events.RepoPreDeleteEvent(repo))
 
         try:
             self.sa.delete(repo)
             if fs_remove:
                 self._delete_filesystem_repo(repo)
             else:
                 log.debug('skipping removal from filesystem')
             old_repo_dict.update({
                 'deleted_by': cur_user,
                 'deleted_on': time.time(),
             })
             if call_events:
                 hooks_base.delete_repository(**old_repo_dict)
                 events.trigger(events.RepoDeleteEvent(repo))
         except Exception:
             log.error(traceback.format_exc())
             raise
 
         return True
 
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         action_logger_generic(
             'granted permission: {} to user: {} on repo: {}'.format(
                 perm, user, repo), namespace='security.repo')
         return obj
 
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
 
         user = self._get_user(user)
         repo = self._get_repo(repo)
 
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
             action_logger_generic(
                 'revoked permission from user: {} on repo: {}'.format(
                     user, repo), namespace='security.repo')
 
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
 
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
 
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         action_logger_generic(
             'granted permission: {} to usergroup: {} on repo: {}'.format(
                 perm, group_name, repo), namespace='security.repo')
 
         return obj
 
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
 
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
             action_logger_generic(
                 'revoked permission from usergroup: {} on repo: {}'.format(
                     group_name, repo), namespace='security.repo')
 
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
 
         :param repo_name:
         """
         repo = self._get_repo(repo_name)
         try:
             obj = self.sa.query(Statistics) \
                 .filter(Statistics.repository == repo).scalar()
             if obj:
                 self.sa.delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def add_repo_field(self, repo_name, field_key, field_label, field_value='',
                        field_type='str', field_desc=''):
 
         repo = self._get_repo(repo_name)
 
         new_field = RepositoryField()
         new_field.repository = repo
         new_field.field_key = field_key
         new_field.field_type = field_type  # python type
         new_field.field_value = field_value
         new_field.field_desc = field_desc
         new_field.field_label = field_label
         self.sa.add(new_field)
         return new_field
 
     def delete_repo_field(self, repo_name, field_key):
         repo = self._get_repo(repo_name)
         field = RepositoryField.get_by_key_name(field_key, repo)
         if field:
             self.sa.delete(field)
 
     def set_landing_rev(self, repo, landing_rev_name):
         if landing_rev_name.startswith('branch:'):
             landing_rev_name = landing_rev_name.split('branch:')[-1]
         scm_instance = repo.scm_instance()
         if scm_instance:
             return scm_instance._remote.set_head_ref(landing_rev_name)
 
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None,
                                 use_global_config=False, install_hooks=True):
         """
         makes repository on filesystem. It's group aware means it'll create
         a repository within a group, and alter the paths accordingly of
         group location
 
         :param repo_name:
         :param alias:
         :param parent:
         :param clone_uri:
         :param repo_store_location:
         """
         from rhodecode.lib.utils import is_valid_repo, is_valid_repo_group
         from rhodecode.model.scm import ScmModel
 
         if Repository.NAME_SEP in repo_name:
             raise ValueError(
                 'repo_name must not contain groups got `%s`' % repo_name)
 
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
 
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception(f'This path {repo_path} is a valid repository')
 
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception(f'This path {repo_path} is a valid group')
 
         log.info('creating repo %s in %s from url: `%s`',
                  repo_name, safe_str(repo_path),
                  obfuscate_url_pw(clone_uri))
 
         backend = get_backend(repo_type)
 
         config_repo = None if use_global_config else repo_name
         if config_repo and new_parent_path:
             config_repo = Repository.NAME_SEP.join(
                 (new_parent_path, config_repo))
         config = make_db_config(clear_session=False, repo=config_repo)
         config.set('extensions', 'largefiles', '')
 
         # patch and reset hooks section of UI config to not run any
         # hooks on creating remote repo
         config.clear_section('hooks')
 
         # TODO: johbo: Unify this, hardcoded "bare=True" does not look nice
         if repo_type == 'git':
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri, bare=True,
                 with_wire={"cache": False})
         else:
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri,
                 with_wire={"cache": False})
 
         if install_hooks:
             repo.install_hooks()
 
         log.debug('Created repo %s with %s backend',
                   safe_str(repo_name), safe_str(repo_type))
         return repo
 
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
             )
         shutil.move(old_path, new_path)
 
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is actually made by
         added rm__ prefix into dir, and rename internal .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         repo_group = repo.group
         log.info("delete_filesystem_repo: removing repository %s", rm_path)
         # disable hg/git internal that it doesn't get detected as repo
         alias = repo.repo_type
 
         config = make_db_config(clear_session=False)
         config.set('extensions', 'largefiles', '')
         bare = getattr(repo.scm_instance(config=config), 'bare', False)
 
         # skip this for bare git repos
         if not bare:
             # disable VCS repo
             vcs_path = os.path.join(rm_path, '.%s' % alias)
             if os.path.exists(vcs_path):
                 shutil.move(vcs_path, os.path.join(rm_path, 'rm__.%s' % alias))
 
         _now = datetime.datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__{}__{}'.format(_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo_group:
             # if repository is in group, prefix the removal path with the group
             args = repo_group.full_path_splitted + [_d]
             _d = os.path.join(*args)
 
         if os.path.isdir(rm_path):
             shutil.move(rm_path, os.path.join(self.repos_path, _d))
 
         # finally cleanup diff-cache if it exists
         cached_diffs_dir = repo.cached_diffs_dir
         if os.path.isdir(cached_diffs_dir):
             shutil.rmtree(cached_diffs_dir)
 
 
 class ReadmeFinder:
     """
     Utility which knows how to find a readme for a specific commit.
 
     The main idea is that this is a configurable algorithm. When creating an
     instance you can define parameters, currently only the `default_renderer`.
     Based on this configuration the method :meth:`search` behaves slightly
     different.
     """
 
     readme_re = re.compile(br'^readme(\.[^.]+)?$', re.IGNORECASE)
     path_re = re.compile(br'^docs?', re.IGNORECASE)
 
     default_priorities = {
         None:        0,
         b'.rst':      1,
         b'.md':       1,
         b'.rest':     2,
         b'.mkdn':     2,
         b'.text':     2,
         b'.txt':      3,
         b'.mdown':    3,
         b'.markdown': 4,
     }
 
     path_priority = {
         b'doc':  0,
         b'docs': 1,
     }
 
     FALLBACK_PRIORITY = 99
 
     RENDERER_TO_EXTENSION = {
         'rst': [b'.rst', b'.rest'],
         'markdown': [b'.md', b'mkdn', b'.mdown', b'.markdown'],
     }
 
     def __init__(self, default_renderer=None):
         self._default_renderer = default_renderer
         self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(default_renderer, [])
 
-    def search(self, commit, path=b'/'):
+    def search(self, commit, path=b'/', nodes=None):
         """
         Find a readme in the given `commit`.
         """
         # firstly, check the PATH type if it is actually a DIR
         bytes_path = safe_bytes(path)
         if commit.get_node(bytes_path).kind != NodeKind.DIR:
             return None
 
-        nodes = commit.get_nodes(bytes_path)
+        if not nodes:
+            nodes = commit.get_nodes(bytes_path)
+
         matches = self._match_readmes(nodes)
         matches = self._sort_according_to_priority(matches)
         if matches:
             return matches[0].node
 
         paths = self._match_paths(nodes)
         paths = self._sort_paths_according_to_priority(paths)
         for bytes_path in paths:
             match = self.search(commit, path=bytes_path)
             if match:
                 return match
 
         return None
 
     def _match_readmes(self, nodes):
         for node in nodes:
             if not node.is_file():
                 continue
             path = node.bytes_path.rsplit(b'/', 1)[-1]
             match = self.readme_re.match(path)
             if match:
                 extension = match.group(1)
                 yield ReadmeMatch(node, match, self._priority(extension))
 
     def _match_paths(self, nodes):
         for node in nodes:
             if not node.is_dir():
                 continue
             match = self.path_re.match(node.bytes_path)
             if match:
                 yield node.bytes_path
 
     def _priority(self, extension):
         renderer_priority = 0 if extension in self._renderer_extensions else 1
         extension_priority = self.default_priorities.get(extension, self.FALLBACK_PRIORITY)
         return renderer_priority, extension_priority
 
     def _sort_according_to_priority(self, matches):
 
         def priority_and_path(match):
             return match.priority, match.path
 
         return sorted(matches, key=priority_and_path)
 
     def _sort_paths_according_to_priority(self, paths):
 
         def priority_and_path(path):
             return self.path_priority.get(path, self.FALLBACK_PRIORITY), path
 
         return sorted(paths, key=priority_and_path)
 
 
 class ReadmeMatch:
 
     def __init__(self, node, match, priority):
         self.node = node
         self._match = match
         self.priority = priority
 
     @property
     def path(self):
         return self.node.path
 
     def __repr__(self):
         return f'<ReadmeMatch {self.path} priority={self.priority}'

rhodecode/templates/files/files_browser_tree.mako

@@ -1,116 +1,118
 <%namespace name="base" file="/base/base.mako"/>
 
 <%doc>
 Please note the content of this file is cached, so changes here might not be reflected when editing.
 add ?no-cache=true into the file url to disable caches.
 
 e.g
 http://docker-dev:10020/ipython/files/master/IPython/frontend/html/notebook/static?no-cache=1
 
 </%doc>
 <%
     at_ref = request.GET.get('at')
     if at_ref:
         query={'at': at_ref}
         default_landing_ref = at_ref or c.rhodecode_db_repo.landing_ref_name
     else:
         query=None
         default_landing_ref = c.commit.raw_id
 %>
 <div id="file-tree-wrapper" class="browser-body ${('full-load' if c.full_load else '')}">
     <table class="code-browser rctable table-bordered">
         <thead>
             <tr>
                 <th>${_('Name')}</th>
                 <th>${_('Size')}</th>
                 <th>${_('Modified')}</th>
                 <th>${_('Last Commit')}</th>
                 <th>${_('Author')}</th>
             </tr>
         </thead>
 
         <tbody id="tbody">
         <tr>
             <td colspan="5">
                 ${h.files_breadcrumbs(c.repo_name, c.rhodecode_db_repo.repo_type, c.commit.raw_id, c.file.path, c.rhodecode_db_repo.landing_ref_name, request.GET.get('at'), limit_items=True)}
             </td>
         </tr>
 
         <% has_files = False %>
         % if not c.file.is_submodule():
-            % for cnt, node in enumerate(c.file):
+            % for cnt, node in enumerate(c.file_nodes):
             <% has_files = True %>
             <tr class="parity${(cnt % 2)}">
                 <td class="td-componentname">
                 % if node.is_submodule():
                   <span class="submodule-dir">
                     % if node.url.startswith('http://') or node.url.startswith('https://'):
                       <a href="${node.url}">
                           <i class="icon-directory browser-dir"></i><span class="tooltip-hovercard" data-hovercard-alt="${node.url}" data-hovercard-url="javascript:renderTemplate('submoduleHovercard', {'submodule_url':'${node.url}'})">${node.name}</span>
                       </a>
                     % else:
                       <i class="icon-directory browser-dir"></i><span class="tooltip-hovercard" data-hovercard-alt="${node.url}" data-hovercard-url="javascript:renderTemplate('submoduleHovercard', {'submodule_url':'${node.url}'})">${node.name}</span>
                     % endif
                   </span>
                 % else:
                   <a href="${h.repo_files_by_ref_url(c.repo_name, c.rhodecode_db_repo.repo_type, f_path=h.safe_str(node.path), ref_name=default_landing_ref, commit_id=c.commit.raw_id, query=query)}">
                     <i class="${('icon-file-text browser-file' if node.is_file() else 'icon-directory browser-dir')}"></i>${node.name}
                   </a>
                 % endif
                 </td>
                 %if node.is_file():
                   <td class="td-size" data-attr-name="size">
                       % if c.full_load:
                         <span data-size="${node.size}">${h.format_byte_size_binary(node.size)}</span>
                       % else:
                         ${_('Loading ...')}
                       % endif
                   </td>
                   <td class="td-time" data-attr-name="modified_at">
                       % if c.full_load:
                         <span data-date="${node.last_commit.date}">${h.age_component(node.last_commit.date)}</span>
                       % endif
                   </td>
                   <td class="td-hash" data-attr-name="commit_id">
                       % if c.full_load:
                       <div class="tooltip-hovercard" data-hovercard-alt="${node.last_commit.message}" data-hovercard-url="${h.route_path('hovercard_repo_commit', repo_name=c.repo_name, commit_id=node.last_commit.raw_id)}">
                           <pre data-commit-id="${node.last_commit.raw_id}">r${node.last_commit.idx}:${node.last_commit.short_id}</pre>
                       </div>
                       % endif
                   </td>
                   <td class="td-user" data-attr-name="author">
                       % if c.full_load:
                       <span data-author="${node.last_commit.author}">${h.gravatar_with_user(request, node.last_commit.author, tooltip=True)|n}</span>
                       % endif
                   </td>
                 %else:
                   <td></td>
                   <td></td>
                   <td></td>
                   <td></td>
                 %endif
               </tr>
             % endfor
         % endif
 
         % if not has_files:
         <tr>
             <td colspan="5">
                 ##empty-dir mostly SVN
 
                 ## submodule if we somehow endup
                 % if c.file.is_submodule():
                     <span class="submodule-dir">
                         <strong>Submodule Node</strong><br/>
                         ${h.escape(c.file.name)}
                         <pre>${c.file.url}</pre>
                     </span>
+                % else:
+                    <div>${_('Empty directory')}</div>
                 %endif
             </td>
         </tr>
         % endif
 
         </tbody>
         <tbody id="tbody_filtered"></tbody>
     </table>
 </div>

rhodecode/tests/vcs/test_git.py

@@ -1,1249 +1,1251
 # Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import datetime
 import mock
 import os
 import sys
 import shutil
 
 import pytest
 
 from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.vcs.backends.base import Reference
 from rhodecode.lib.vcs.backends.git import GitRepository, GitCommit, discover_git_version
 from rhodecode.lib.vcs.exceptions import RepositoryError, VCSError, NodeDoesNotExistError
 from rhodecode.lib.vcs.nodes import NodeKind, FileNode, DirNode, NodeState, SubModuleNode, RootNode
 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
 from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 pytestmark = pytest.mark.backends("git")
 
 
 DIFF_FROM_REMOTE = rb"""diff --git a/foobar b/foobar
 new file mode 100644
 index 0000000..f6ea049
 --- /dev/null
 +++ b/foobar
 @@ -0,0 +1 @@
 +foobar
 \ No newline at end of file
 diff --git a/foobar2 b/foobar2
 new file mode 100644
 index 0000000..e8c9d6b
 --- /dev/null
 +++ b/foobar2
 @@ -0,0 +1 @@
 +foobar2
 \ No newline at end of file
 """
 
 
 def callable_get_diff(*args, **kwargs):
     return DIFF_FROM_REMOTE
 
 
 class TestGitRepository(object):
     @pytest.fixture(autouse=True)
     def prepare(self, request, baseapp):
         self.repo = GitRepository(TEST_GIT_REPO, bare=True)
         self.repo.count()
 
     def get_clone_repo(self, tmpdir):
         """
         Return a non bare clone of the base repo.
         """
         clone_path = str(tmpdir.join("clone-repo"))
         repo_clone = GitRepository(clone_path, create=True, src_url=self.repo.path, bare=False)
 
         return repo_clone
 
     def get_empty_repo(self, tmpdir, bare=False):
         """
         Return a non bare empty repo.
         """
         clone_path = str(tmpdir.join("empty-repo"))
         return GitRepository(clone_path, create=True, bare=bare)
 
     def test_wrong_repo_path(self):
         wrong_repo_path = "/tmp/errorrepo_git"
         with pytest.raises(RepositoryError):
             GitRepository(wrong_repo_path)
 
     def test_repo_clone(self, tmp_path_factory):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = f"{tmp_path_factory.mktemp('_')}_{TEST_GIT_REPO_CLONE}"
         repo_clone = GitRepository(clone_path, src_url=TEST_GIT_REPO, create=True, do_workspace_checkout=True)
 
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
         # Checking hashes of commits should be enough
         for commit in repo.get_commits():
             raw_id = commit.raw_id
             assert raw_id == repo_clone.get_commit(raw_id).raw_id
 
     def test_repo_clone_without_create(self):
         with pytest.raises(RepositoryError):
             GitRepository(TEST_GIT_REPO_CLONE + "_wo_create", src_url=TEST_GIT_REPO)
 
     def test_repo_clone_with_update(self, tmp_path_factory):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = "{}_{}_update".format(tmp_path_factory.mktemp("_"), TEST_GIT_REPO_CLONE)
 
         repo_clone = GitRepository(clone_path, create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=True)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
 
         # check if current workdir was updated
         fpath = os.path.join(clone_path, "MANIFEST.in")
         assert os.path.isfile(fpath)
 
     def test_repo_clone_without_update(self, tmp_path_factory):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = "{}_{}_without_update".format(tmp_path_factory.mktemp("_"), TEST_GIT_REPO_CLONE)
         repo_clone = GitRepository(clone_path, create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=False)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
         # check if current workdir was *NOT* updated
         fpath = os.path.join(clone_path, "MANIFEST.in")
         # Make sure it's not bare repo
         assert not repo_clone.bare
         assert not os.path.isfile(fpath)
 
     def test_repo_clone_into_bare_repo(self, tmp_path_factory):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = "{}_{}_bare.git".format(tmp_path_factory.mktemp("_"), TEST_GIT_REPO_CLONE)
         repo_clone = GitRepository(clone_path, create=True, src_url=repo.path, bare=True)
         assert repo_clone.bare
 
     def test_create_repo_is_not_bare_by_default(self):
         repo = GitRepository(get_new_dir("not-bare-by-default"), create=True)
         assert not repo.bare
 
     def test_create_bare_repo(self):
         repo = GitRepository(get_new_dir("bare-repo"), create=True, bare=True)
         assert repo.bare
 
     def test_update_server_info(self):
         self.repo._update_server_info()
 
     def test_fetch(self, vcsbackend_git):
         # Note: This is a git specific part of the API, it's only implemented
         # by the git backend.
         source_repo = vcsbackend_git.repo
         target_repo = vcsbackend_git.create_repo(bare=True)
         target_repo.fetch(source_repo.path)
         # Note: Get a fresh instance, avoids caching trouble
         target_repo = vcsbackend_git.backend(target_repo.path)
         assert len(source_repo.commit_ids) == len(target_repo.commit_ids)
 
     def test_commit_ids(self):
         # there are 112 commits (by now)
         # so we can assume they would be available from now on
         subset = {
             "c1214f7e79e02fc37156ff215cd71275450cffc3",
             "38b5fe81f109cb111f549bfe9bb6b267e10bc557",
             "fa6600f6848800641328adbf7811fd2372c02ab2",
             "102607b09cdd60e2793929c4f90478be29f85a17",
             "49d3fd156b6f7db46313fac355dca1a0b94a0017",
             "2d1028c054665b962fa3d307adfc923ddd528038",
             "d7e0d30fbcae12c90680eb095a4f5f02505ce501",
             "ff7ca51e58c505fec0dd2491de52c622bb7a806b",
             "dd80b0f6cf5052f17cc738c2951c4f2070200d7f",
             "8430a588b43b5d6da365400117c89400326e7992",
             "d955cd312c17b02143c04fa1099a352b04368118",
             "f67b87e5c629c2ee0ba58f85197e423ff28d735b",
             "add63e382e4aabc9e1afdc4bdc24506c269b7618",
             "f298fe1189f1b69779a4423f40b48edf92a703fc",
             "bd9b619eb41994cac43d67cf4ccc8399c1125808",
             "6e125e7c890379446e98980d8ed60fba87d0f6d1",
             "d4a54db9f745dfeba6933bf5b1e79e15d0af20bd",
             "0b05e4ed56c802098dfc813cbe779b2f49e92500",
             "191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e",
             "45223f8f114c64bf4d6f853e3c35a369a6305520",
             "ca1eb7957a54bce53b12d1a51b13452f95bc7c7e",
             "f5ea29fc42ef67a2a5a7aecff10e1566699acd68",
             "27d48942240f5b91dfda77accd2caac94708cc7d",
             "622f0eb0bafd619d2560c26f80f09e3b0b0d78af",
             "e686b958768ee96af8029fe19c6050b1a8dd3b2b",
         }
         assert subset.issubset(set(self.repo.commit_ids))
 
     def test_slicing(self):
         # 4 1 5 10 95
         for sfrom, sto, size in [(0, 4, 4), (1, 2, 1), (10, 15, 5), (10, 20, 10), (5, 100, 95)]:
             commit_ids = list(self.repo[sfrom:sto])
             assert len(commit_ids) == size
             assert commit_ids[0] == self.repo.get_commit(commit_idx=sfrom)
             assert commit_ids[-1] == self.repo.get_commit(commit_idx=sto - 1)
 
     def test_branches(self):
         # TODO: Need more tests here
         # Removed (those are 'remotes' branches for cloned repo)
         # assert 'master' in self.repo.branches
         # assert 'gittree' in self.repo.branches
         # assert 'web-branch' in self.repo.branches
         for __, commit_id in self.repo.branches.items():
             assert isinstance(self.repo.get_commit(commit_id), GitCommit)
 
     def test_tags(self):
         # TODO: Need more tests here
         assert "v0.1.1" in self.repo.tags
         assert "v0.1.2" in self.repo.tags
         for __, commit_id in self.repo.tags.items():
             assert isinstance(self.repo.get_commit(commit_id), GitCommit)
 
     def _test_single_commit_cache(self, commit_id):
         commit = self.repo.get_commit(commit_id)
         assert commit_id in self.repo.commits
         assert commit is self.repo.commits[commit_id]
 
     def test_initial_commit(self):
         commit_id = self.repo.commit_ids[0]
         init_commit = self.repo.get_commit(commit_id)
         init_author = init_commit.author
 
         assert init_commit.message == "initial import\n"
         assert init_author == "Marcin Kuzminski <[e-mail]>"
         assert init_author == init_commit.committer
         assert sorted(init_commit.added_paths) == sorted(
             [
                 b"vcs/__init__.py",
                 b"vcs/backends/BaseRepository.py",
                 b"vcs/backends/__init__.py",
             ]
         )
         assert sorted(init_commit.affected_files) == sorted(
             [
                 b"vcs/__init__.py",
                 b"vcs/backends/BaseRepository.py",
                 b"vcs/backends/__init__.py",
             ]
         )
 
         for path in (b"vcs/__init__.py", b"vcs/backends/BaseRepository.py", b"vcs/backends/__init__.py"):
             assert isinstance(init_commit.get_node(path), FileNode)
         for path in (b"", b"vcs", b"vcs/backends"):
             assert isinstance(init_commit.get_node(path), DirNode)
 
         with pytest.raises(NodeDoesNotExistError):
             init_commit.get_node(path=b"foobar")
 
         node = init_commit.get_node(b"vcs/")
         assert hasattr(node, "kind")
         assert node.kind == NodeKind.DIR
 
         node = init_commit.get_node(b"vcs")
         assert hasattr(node, "kind")
         assert node.kind == NodeKind.DIR
 
         node = init_commit.get_node(b"vcs/__init__.py")
         assert hasattr(node, "kind")
         assert node.kind == NodeKind.FILE
 
     def test_not_existing_commit(self):
         with pytest.raises(RepositoryError):
             self.repo.get_commit("f" * 40)
 
     def test_commit10(self):
         commit10 = self.repo.get_commit(self.repo.commit_ids[9])
         README = """===
 VCS
 ===
 
 Various Version Control System management abstraction layer for Python.
 
 Introduction
 ------------
 
 TODO: To be written...
 
 """
         node = commit10.get_node(b"README.rst")
         assert node.kind == NodeKind.FILE
         assert node.str_content == README
 
     def test_head(self):
         assert self.repo.head == self.repo.get_commit().raw_id
 
     def test_checkout_with_create(self, tmpdir):
         repo_clone = self.get_clone_repo(tmpdir)
 
         new_branch = "new_branch"
         assert repo_clone._current_branch() == "master"
         assert set(repo_clone.branches) == {"master"}
         repo_clone._checkout(new_branch, create=True)
 
         # Branches is a lazy property so we need to recrete the Repo object.
         repo_clone = GitRepository(repo_clone.path)
         assert set(repo_clone.branches) == {"master", new_branch}
         assert repo_clone._current_branch() == new_branch
 
     def test_checkout(self, tmpdir):
         repo_clone = self.get_clone_repo(tmpdir)
 
         repo_clone._checkout("new_branch", create=True)
         repo_clone._checkout("master")
 
         assert repo_clone._current_branch() == "master"
 
     def test_checkout_same_branch(self, tmpdir):
         repo_clone = self.get_clone_repo(tmpdir)
 
         repo_clone._checkout("master")
         assert repo_clone._current_branch() == "master"
 
     def test_checkout_branch_already_exists(self, tmpdir):
         repo_clone = self.get_clone_repo(tmpdir)
 
         with pytest.raises(RepositoryError):
             repo_clone._checkout("master", create=True)
 
     def test_checkout_bare_repo(self):
         with pytest.raises(RepositoryError):
             self.repo._checkout("master")
 
     def test_current_branch_bare_repo(self):
         with pytest.raises(RepositoryError):
             self.repo._current_branch()
 
     def test_current_branch_empty_repo(self, tmpdir):
         repo = self.get_empty_repo(tmpdir)
         assert repo._current_branch() is None
 
     def test_local_clone(self, tmp_path_factory):
         clone_path = str(tmp_path_factory.mktemp("test-local-clone"))
         self.repo._local_clone(clone_path, "master")
         repo_clone = GitRepository(clone_path)
 
         assert self.repo.commit_ids == repo_clone.commit_ids
 
     def test_local_clone_with_specific_branch(self, tmpdir):
         source_repo = self.get_clone_repo(tmpdir)
 
         # Create a new branch in source repo
         new_branch_commit = source_repo.commit_ids[-3]
         source_repo._checkout(new_branch_commit)
         source_repo._checkout("new_branch", create=True)
 
         clone_path = str(tmpdir.join("git-clone-path-1"))
         source_repo._local_clone(clone_path, "new_branch")
         repo_clone = GitRepository(clone_path)
 
         assert source_repo.commit_ids[: -3 + 1] == repo_clone.commit_ids
 
         clone_path = str(tmpdir.join("git-clone-path-2"))
         source_repo._local_clone(clone_path, "master")
         repo_clone = GitRepository(clone_path)
 
         assert source_repo.commit_ids == repo_clone.commit_ids
 
     def test_local_clone_fails_if_target_exists(self):
         with pytest.raises(RepositoryError):
             self.repo._local_clone(self.repo.path, "master")
 
     def test_local_fetch(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
         source_repo = self.get_clone_repo(tmpdir)
 
         # Create a new branch in source repo
         master_commit = source_repo.commit_ids[-1]
         new_branch_commit = source_repo.commit_ids[-3]
         source_repo._checkout(new_branch_commit)
         source_repo._checkout("new_branch", create=True)
 
         target_repo._local_fetch(source_repo.path, "new_branch")
         assert target_repo._last_fetch_heads() == [new_branch_commit]
 
         target_repo._local_fetch(source_repo.path, "master")
         assert target_repo._last_fetch_heads() == [master_commit]
 
     def test_local_fetch_from_bare_repo(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
         target_repo._local_fetch(self.repo.path, "master")
 
         master_commit = self.repo.commit_ids[-1]
         assert target_repo._last_fetch_heads() == [master_commit]
 
     def test_local_fetch_from_same_repo(self):
         with pytest.raises(ValueError):
             self.repo._local_fetch(self.repo.path, "master")
 
     def test_local_fetch_branch_does_not_exist(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
 
         with pytest.raises(RepositoryError):
             target_repo._local_fetch(self.repo.path, "new_branch")
 
     def test_local_pull(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
         source_repo = self.get_clone_repo(tmpdir)
 
         # Create a new branch in source repo
         master_commit = source_repo.commit_ids[-1]
         new_branch_commit = source_repo.commit_ids[-3]
         source_repo._checkout(new_branch_commit)
         source_repo._checkout("new_branch", create=True)
 
         target_repo._local_pull(source_repo.path, "new_branch")
         target_repo = GitRepository(target_repo.path)
         assert target_repo.head == new_branch_commit
 
         target_repo._local_pull(source_repo.path, "master")
         target_repo = GitRepository(target_repo.path)
         assert target_repo.head == master_commit
 
     def test_local_pull_in_bare_repo(self):
         with pytest.raises(RepositoryError):
             self.repo._local_pull(self.repo.path, "master")
 
     def test_local_merge(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
         source_repo = self.get_clone_repo(tmpdir)
 
         # Create a new branch in source repo
         master_commit = source_repo.commit_ids[-1]
         new_branch_commit = source_repo.commit_ids[-3]
         source_repo._checkout(new_branch_commit)
         source_repo._checkout("new_branch", create=True)
 
         # This is required as one cannot do a -ff-only merge in an empty repo.
         target_repo._local_pull(source_repo.path, "new_branch")
 
         target_repo._local_fetch(source_repo.path, "master")
         merge_message = "Merge message\n\nDescription:..."
         user_name = "Albert Einstein"
         user_email = "[e-mail]"
         target_repo._local_merge(merge_message, user_name, user_email, target_repo._last_fetch_heads())
 
         target_repo = GitRepository(target_repo.path)
         assert target_repo.commit_ids[-2] == master_commit
         last_commit = target_repo.get_commit(target_repo.head)
         assert last_commit.message.strip() == merge_message
         assert last_commit.author == "%s <%s>" % (user_name, user_email)
 
         assert not os.path.exists(os.path.join(target_repo.path, ".git", "MERGE_HEAD"))
 
     def test_local_merge_raises_exception_on_conflict(self, vcsbackend_git):
         target_repo = vcsbackend_git.create_repo(number_of_commits=1)
         vcsbackend_git.ensure_file(b"README", b"I will conflict with you!!!")
 
         target_repo._local_fetch(self.repo.path, "master")
         with pytest.raises(RepositoryError):
             target_repo._local_merge("merge_message", "user name", "[e-mail]", target_repo._last_fetch_heads())
 
         # Check we are not left in an intermediate merge state
         assert not os.path.exists(os.path.join(target_repo.path, ".git", "MERGE_HEAD"))
 
     def test_local_merge_into_empty_repo(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
 
         # This is required as one cannot do a -ff-only merge in an empty repo.
         target_repo._local_fetch(self.repo.path, "master")
         with pytest.raises(RepositoryError):
             target_repo._local_merge("merge_message", "user name", "[e-mail]", target_repo._last_fetch_heads())
 
     def test_local_merge_in_bare_repo(self):
         with pytest.raises(RepositoryError):
             self.repo._local_merge("merge_message", "user name", "[e-mail]", None)
 
     def test_local_push_non_bare(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
 
         pushed_branch = "pushed_branch"
         self.repo._local_push("master", target_repo.path, pushed_branch)
         # Fix the HEAD of the target repo, or otherwise GitRepository won't
         # report any branches.
         with open(os.path.join(target_repo.path, ".git", "HEAD"), "w") as f:
             f.write("ref: refs/heads/%s" % pushed_branch)
 
         target_repo = GitRepository(target_repo.path)
 
         assert target_repo.branches[pushed_branch] == self.repo.branches["master"]
 
     def test_local_push_bare(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir, bare=True)
 
         pushed_branch = "pushed_branch"
         self.repo._local_push("master", target_repo.path, pushed_branch)
         # Fix the HEAD of the target repo, or otherwise GitRepository won't
         # report any branches.
         with open(os.path.join(target_repo.path, "HEAD"), "w") as f:
             f.write("ref: refs/heads/%s" % pushed_branch)
 
         target_repo = GitRepository(target_repo.path)
 
         assert target_repo.branches[pushed_branch] == self.repo.branches["master"]
 
     def test_local_push_non_bare_target_branch_is_checked_out(self, tmpdir):
         target_repo = self.get_clone_repo(tmpdir)
 
         pushed_branch = "pushed_branch"
         # Create a new branch in source repo
         new_branch_commit = target_repo.commit_ids[-3]
         target_repo._checkout(new_branch_commit)
         target_repo._checkout(pushed_branch, create=True)
 
         self.repo._local_push("master", target_repo.path, pushed_branch)
 
         target_repo = GitRepository(target_repo.path)
 
         assert target_repo.branches[pushed_branch] == self.repo.branches["master"]
 
     def test_local_push_raises_exception_on_conflict(self, vcsbackend_git):
         target_repo = vcsbackend_git.create_repo(number_of_commits=1)
         with pytest.raises(RepositoryError):
             self.repo._local_push("master", target_repo.path, "master")
 
     def test_hooks_can_be_enabled_via_env_variable_for_local_push(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir, bare=True)
 
         with mock.patch.object(self.repo, "run_git_command") as run_mock:
             self.repo._local_push("master", target_repo.path, "master", enable_hooks=True)
         env = run_mock.call_args[1]["extra_env"]
         assert "RC_SKIP_HOOKS" not in env
 
     def _add_failing_hook(self, repo_path, hook_name, bare=False):
         path_components = ["hooks", hook_name] if bare else [".git", "hooks", hook_name]
         hook_path = os.path.join(repo_path, *path_components)
         with open(hook_path, "w") as f:
             script_lines = [
                 "#!%s" % sys.executable,
                 "import os",
                 "import sys",
                 'if os.environ.get("RC_SKIP_HOOKS"):',
                 "    sys.exit(0)",
                 "sys.exit(1)",
             ]
             f.write("\n".join(script_lines))
         os.chmod(hook_path, 0o755)
 
     def test_local_push_does_not_execute_hook(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir)
 
         pushed_branch = "pushed_branch"
         self._add_failing_hook(target_repo.path, "pre-receive")
         self.repo._local_push("master", target_repo.path, pushed_branch)
         # Fix the HEAD of the target repo, or otherwise GitRepository won't
         # report any branches.
         with open(os.path.join(target_repo.path, ".git", "HEAD"), "w") as f:
             f.write("ref: refs/heads/%s" % pushed_branch)
 
         target_repo = GitRepository(target_repo.path)
 
         assert target_repo.branches[pushed_branch] == self.repo.branches["master"]
 
     def test_local_push_executes_hook(self, tmpdir):
         target_repo = self.get_empty_repo(tmpdir, bare=True)
         self._add_failing_hook(target_repo.path, "pre-receive", bare=True)
         with pytest.raises(RepositoryError):
             self.repo._local_push("master", target_repo.path, "master", enable_hooks=True)
 
     def test_maybe_prepare_merge_workspace(self):
         workspace = self.repo._maybe_prepare_merge_workspace(
             2, "pr2", Reference("branch", "master", "unused"), Reference("branch", "master", "unused")
         )
 
         assert os.path.isdir(workspace)
         workspace_repo = GitRepository(workspace)
         assert workspace_repo.branches == self.repo.branches
 
         # Calling it a second time should also succeed
         workspace = self.repo._maybe_prepare_merge_workspace(
             2, "pr2", Reference("branch", "master", "unused"), Reference("branch", "master", "unused")
         )
         assert os.path.isdir(workspace)
 
     def test_maybe_prepare_merge_workspace_different_refs(self):
         workspace = self.repo._maybe_prepare_merge_workspace(
             2, "pr2", Reference("branch", "master", "unused"), Reference("branch", "develop", "unused")
         )
 
         assert os.path.isdir(workspace)
         workspace_repo = GitRepository(workspace)
         assert workspace_repo.branches == self.repo.branches
 
         # Calling it a second time should also succeed
         workspace = self.repo._maybe_prepare_merge_workspace(
             2, "pr2", Reference("branch", "master", "unused"), Reference("branch", "develop", "unused")
         )
         assert os.path.isdir(workspace)
 
     def test_cleanup_merge_workspace(self):
         workspace = self.repo._maybe_prepare_merge_workspace(
             2, "pr3", Reference("branch", "master", "unused"), Reference("branch", "master", "unused")
         )
         self.repo.cleanup_merge_workspace(2, "pr3")
 
         assert not os.path.exists(workspace)
 
     def test_cleanup_merge_workspace_invalid_workspace_id(self):
         # No assert: because in case of an inexistent workspace this function
         # should still succeed.
         self.repo.cleanup_merge_workspace(1, "pr4")
 
     def test_set_refs(self):
         test_ref = "refs/test-refs/abcde"
         test_commit_id = "ecb86e1f424f2608262b130db174a7dfd25a6623"
 
         self.repo.set_refs(test_ref, test_commit_id)
         stdout, _ = self.repo.run_git_command(["show-ref"])
         assert test_ref in stdout
         assert test_commit_id in stdout
 
     def test_remove_ref(self):
         test_ref = "refs/test-refs/abcde"
         test_commit_id = "ecb86e1f424f2608262b130db174a7dfd25a6623"
         self.repo.set_refs(test_ref, test_commit_id)
         stdout, _ = self.repo.run_git_command(["show-ref"])
         assert test_ref in stdout
         assert test_commit_id in stdout
 
         self.repo.remove_ref(test_ref)
         stdout, _ = self.repo.run_git_command(["show-ref"])
         assert test_ref not in stdout
         assert test_commit_id not in stdout
 
 
 class TestGitCommit(object):
     @pytest.fixture(autouse=True)
     def prepare(self):
         self.repo = GitRepository(TEST_GIT_REPO)
 
     def test_default_commit(self):
         tip = self.repo.get_commit()
         assert tip == self.repo.get_commit(None)
         assert tip == self.repo.get_commit("tip")
 
     def test_root_node(self):
         tip = self.repo.get_commit()
         assert tip.root is tip.get_node(b"")
 
     def test_lazy_fetch(self):
         """
         Test if commit's nodes expands and are cached as we walk through
         the commit. This test is somewhat hard to write as order of tests
         is a key here. Written by running command after command in a shell.
         """
         commit_id = "2a13f185e4525f9d4b59882791a2d397b90d5ddc"
         assert commit_id in self.repo.commit_ids
         commit = self.repo.get_commit(commit_id)
         assert len(commit.nodes) == 0
         root = commit.root
         assert len(commit.nodes) == 1
         assert len(root.nodes) == 8
         # accessing root.nodes updates commit.nodes
         assert len(commit.nodes) == 9
 
         docs = commit.get_node(b"docs")
         # we haven't yet accessed anything new as docs dir was already cached
         assert len(commit.nodes) == 9
         assert len(docs.nodes) == 8
         # accessing docs.nodes updates commit.nodes
         assert len(commit.nodes) == 17
 
         assert docs is commit.get_node(b"docs")
         assert docs is root.nodes[0]
         assert docs is root.dirs[0]
         assert docs is commit.get_node(b"docs")
 
     def test_nodes_with_commit(self):
         commit_id = "2a13f185e4525f9d4b59882791a2d397b90d5ddc"
         commit = self.repo.get_commit(commit_id)
         root = commit.root
         assert isinstance(root, RootNode)
         docs = commit.get_node(b"docs")
         assert docs is commit.get_node(b"docs")
         api = commit.get_node(b"docs/api")
         assert api is commit.get_node(b"docs/api")
         index = commit.get_node(b"docs/api/index.rst")
         assert index is commit.get_node(b"docs/api/index.rst")
 
     def test_branch_and_tags(self):
         """
         rev0 = self.repo.commit_ids[0]
         commit0 = self.repo.get_commit(rev0)
         assert commit0.branch == 'master'
         assert commit0.tags == []
 
         rev10 = self.repo.commit_ids[10]
         commit10 = self.repo.get_commit(rev10)
         assert commit10.branch == 'master'
         assert commit10.tags == []
 
         rev44 = self.repo.commit_ids[44]
         commit44 = self.repo.get_commit(rev44)
         assert commit44.branch == 'web-branch'
 
         tip = self.repo.get_commit('tip')
         assert 'tip' in tip.tags
         """
         # Those tests would fail - branches are now going
         # to be changed at main API in order to support git backend
         pass
 
     def test_file_size(self):
         to_check = (
             ("c1214f7e79e02fc37156ff215cd71275450cffc3", b"vcs/backends/BaseRepository.py", 502),
             ("d7e0d30fbcae12c90680eb095a4f5f02505ce501", b"vcs/backends/hg.py", 854),
             ("6e125e7c890379446e98980d8ed60fba87d0f6d1", b"setup.py", 1068),
             ("d955cd312c17b02143c04fa1099a352b04368118", b"vcs/backends/base.py", 2921),
             ("ca1eb7957a54bce53b12d1a51b13452f95bc7c7e", b"vcs/backends/base.py", 3936),
             ("f50f42baeed5af6518ef4b0cb2f1423f3851a941", b"vcs/backends/base.py", 6189),
         )
         for commit_id, path, size in to_check:
             node = self.repo.get_commit(commit_id).get_node(path)
             assert node.is_file()
             assert node.size == size
 
     def test_file_history_from_commits(self):
         node = self.repo[10].get_node(b"setup.py")
         commit_ids = [commit.raw_id for commit in node.history]
         assert ["ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == commit_ids
 
         node = self.repo[20].get_node(b"setup.py")
         node_ids = [commit.raw_id for commit in node.history]
         assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids
 
         # special case we check history from commit that has this particular
         # file changed this means we check if it's included as well
         node = self.repo.get_commit("191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e").get_node(b"setup.py")
         node_ids = [commit.raw_id for commit in node.history]
         assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids
 
     def test_file_history(self):
         # we can only check if those commits are present in the history
         # as we cannot update this test every time file is changed
         files = {
             b"setup.py": [
                 "54386793436c938cff89326944d4c2702340037d",
                 "51d254f0ecf5df2ce50c0b115741f4cf13985dab",
                 "998ed409c795fec2012b1c0ca054d99888b22090",
                 "5e0eb4c47f56564395f76333f319d26c79e2fb09",
                 "0115510b70c7229dbc5dc49036b32e7d91d23acd",
                 "7cb3fd1b6d8c20ba89e2264f1c8baebc8a52d36e",
                 "2a13f185e4525f9d4b59882791a2d397b90d5ddc",
                 "191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e",
                 "ff7ca51e58c505fec0dd2491de52c622bb7a806b",
             ],
             b"vcs/nodes.py": [
                 "33fa3223355104431402a888fa77a4e9956feb3e",
                 "fa014c12c26d10ba682fadb78f2a11c24c8118e1",
                 "e686b958768ee96af8029fe19c6050b1a8dd3b2b",
                 "ab5721ca0a081f26bf43d9051e615af2cc99952f",
                 "c877b68d18e792a66b7f4c529ea02c8f80801542",
                 "4313566d2e417cb382948f8d9d7c765330356054",
                 "6c2303a793671e807d1cfc70134c9ca0767d98c2",
                 "54386793436c938cff89326944d4c2702340037d",
                 "54000345d2e78b03a99d561399e8e548de3f3203",
                 "1c6b3677b37ea064cb4b51714d8f7498f93f4b2b",
                 "2d03ca750a44440fb5ea8b751176d1f36f8e8f46",
                 "2a08b128c206db48c2f0b8f70df060e6db0ae4f8",
                 "30c26513ff1eb8e5ce0e1c6b477ee5dc50e2f34b",
                 "ac71e9503c2ca95542839af0ce7b64011b72ea7c",
                 "12669288fd13adba2a9b7dd5b870cc23ffab92d2",
                 "5a0c84f3e6fe3473e4c8427199d5a6fc71a9b382",
                 "12f2f5e2b38e6ff3fbdb5d722efed9aa72ecb0d5",
                 "5eab1222a7cd4bfcbabc218ca6d04276d4e27378",
                 "f50f42baeed5af6518ef4b0cb2f1423f3851a941",
                 "d7e390a45f6aa96f04f5e7f583ad4f867431aa25",
                 "f15c21f97864b4f071cddfbf2750ec2e23859414",
                 "e906ef056cf539a4e4e5fc8003eaf7cf14dd8ade",
                 "ea2b108b48aa8f8c9c4a941f66c1a03315ca1c3b",
                 "84dec09632a4458f79f50ddbbd155506c460b4f9",
                 "0115510b70c7229dbc5dc49036b32e7d91d23acd",
                 "2a13f185e4525f9d4b59882791a2d397b90d5ddc",
                 "3bf1c5868e570e39569d094f922d33ced2fa3b2b",
                 "b8d04012574729d2c29886e53b1a43ef16dd00a1",
                 "6970b057cffe4aab0a792aa634c89f4bebf01441",
                 "dd80b0f6cf5052f17cc738c2951c4f2070200d7f",
                 "ff7ca51e58c505fec0dd2491de52c622bb7a806b",
             ],
             b"vcs/backends/git.py": [
                 "4cf116ad5a457530381135e2f4c453e68a1b0105",
                 "9a751d84d8e9408e736329767387f41b36935153",
                 "cb681fb539c3faaedbcdf5ca71ca413425c18f01",
                 "428f81bb652bcba8d631bce926e8834ff49bdcc6",
                 "180ab15aebf26f98f714d8c68715e0f05fa6e1c7",
                 "2b8e07312a2e89e92b90426ab97f349f4bce2a3a",
                 "50e08c506174d8645a4bb517dd122ac946a0f3bf",
                 "54000345d2e78b03a99d561399e8e548de3f3203",
             ],
         }
         for path, commit_ids in files.items():
             node = self.repo.get_commit(commit_ids[0]).get_node(path)
             node_ids = [commit.raw_id for commit in node.history]
             assert set(commit_ids).issubset(set(node_ids)), (
                 "We assumed that %s is subset of commit_ids for which file %s "
                 "has been changed, and history of that node returned: %s" % (commit_ids, path, node_ids)
             )
 
     def test_file_annotate(self):
         files = {
             b"vcs/backends/__init__.py": {
                 "c1214f7e79e02fc37156ff215cd71275450cffc3": {
                     "lines_no": 1,
                     "commits": [
                         "c1214f7e79e02fc37156ff215cd71275450cffc3",
                     ],
                 },
                 "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647": {
                     "lines_no": 21,
                     "commits": [
                         "49d3fd156b6f7db46313fac355dca1a0b94a0017",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                     ],
                 },
                 "e29b67bd158580fc90fc5e9111240b90e6e86064": {
                     "lines_no": 32,
                     "commits": [
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "5eab1222a7cd4bfcbabc218ca6d04276d4e27378",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "54000345d2e78b03a99d561399e8e548de3f3203",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "78c3f0c23b7ee935ec276acb8b8212444c33c396",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "2a13f185e4525f9d4b59882791a2d397b90d5ddc",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "78c3f0c23b7ee935ec276acb8b8212444c33c396",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "992f38217b979d0b0987d0bae3cc26dac85d9b19",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                         "16fba1ae9334d79b66d7afed2c2dfbfa2ae53647",
                     ],
                 },
             },
         }
 
         for fname, commit_dict in files.items():
             for commit_id, __ in commit_dict.items():
                 commit = self.repo.get_commit(commit_id)
 
                 l1_1 = [x[1] for x in commit.get_file_annotate(fname)]
                 l1_2 = [x[2]().raw_id for x in commit.get_file_annotate(fname)]
                 assert l1_1 == l1_2
                 l1 = l1_1
                 l2 = files[fname][commit_id]["commits"]
                 assert l1 == l2, (
                     "The lists of commit_ids for %s@commit_id %s"
                     "from annotation list should match each other, "
                     "got \n%s \nvs \n%s " % (fname, commit_id, l1, l2)
                 )
 
     def test_files_state(self):
         """
         Tests state of FileNodes.
         """
         commit = self.repo.get_commit("e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0")
         node = commit.get_node(b"vcs/utils/diffs.py")
         assert node.bytes_path in commit.added_paths
 
         commit = self.repo.get_commit("33fa3223355104431402a888fa77a4e9956feb3e")
         node = commit.get_node(b".hgignore")
         assert node.bytes_path in commit.changed_paths
 
         commit = self.repo.get_commit("e29b67bd158580fc90fc5e9111240b90e6e86064")
         node = commit.get_node(b"setup.py")
         assert node.bytes_path not in commit.affected_files
 
         # If node has REMOVED state then trying to fetch it would raise
         # CommitError exception
         commit = self.repo.get_commit("fa6600f6848800641328adbf7811fd2372c02ab2")
         path = b"vcs/backends/BaseRepository.py"
         with pytest.raises(NodeDoesNotExistError):
             commit.get_node(path)
 
         # but it would be one of ``removed`` (commit's attribute)
         assert path in [rf for rf in commit.removed_paths]
 
         commit = self.repo.get_commit("54386793436c938cff89326944d4c2702340037d")
         changed = [b"setup.py", b"tests/test_nodes.py", b"vcs/backends/hg.py", b"vcs/nodes.py"]
         assert set(changed) == set([f for f in commit.changed_paths])
 
     def test_unicode_branch_refs(self):
         unicode_branches = {
             "refs/heads/unicode": "6c0ce52b229aa978889e91b38777f800e85f330b",
             "refs/heads/uniçö∂e": "ürl",
         }
         with mock.patch(("rhodecode.lib.vcs.backends.git.repository" ".GitRepository._refs"), unicode_branches):
             branches = self.repo.branches
 
         assert "unicode" in branches
         assert "uniçö∂e" in branches
 
     def test_unicode_tag_refs(self):
         unicode_tags = {
             "refs/tags/unicode": "6c0ce52b229aa978889e91b38777f800e85f330b",
             "refs/tags/uniçö∂e": "6c0ce52b229aa978889e91b38777f800e85f330b",
         }
         with mock.patch(("rhodecode.lib.vcs.backends.git.repository" ".GitRepository._refs"), unicode_tags):
             tags = self.repo.tags
 
         assert "unicode" in tags
         assert "uniçö∂e" in tags
 
     def test_commit_message_is_unicode(self):
         for commit in self.repo:
             assert type(commit.message) == str
 
     def test_commit_author_is_unicode(self):
         for commit in self.repo:
             assert type(commit.author) == str
 
     def test_repo_files_content_types(self):
         commit = self.repo.get_commit()
         for node in commit.get_node(b"/"):
             if node.is_file():
                 assert type(node.content) == bytes
                 assert type(node.str_content) == str
 
     def test_wrong_path(self):
         # There is 'setup.py' in the root dir but not there:
         path = b"foo/bar/setup.py"
         tip = self.repo.get_commit()
         with pytest.raises(VCSError):
             tip.get_node(path)
 
     @pytest.mark.parametrize(
         "author_email, commit_id",
         [
             ("[e-mail]", "c1214f7e79e02fc37156ff215cd71275450cffc3"),
             ("[e-mail]", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"),
             ("none@none", "8430a588b43b5d6da365400117c89400326e7992"),
         ],
     )
     def test_author_email(self, author_email, commit_id):
         commit = self.repo.get_commit(commit_id)
         assert author_email == commit.author_email
 
     @pytest.mark.parametrize(
         "author, commit_id",
         [
             ("Marcin Kuzminski", "c1214f7e79e02fc37156ff215cd71275450cffc3"),
             ("Lukasz Balcerzak", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"),
             ("marcink", "8430a588b43b5d6da365400117c89400326e7992"),
         ],
     )
     def test_author_username(self, author, commit_id):
         commit = self.repo.get_commit(commit_id)
         assert author == commit.author_name
 
 
 class TestLargeFileRepo(object):
     def test_large_file(self, backend_git):
         conf = make_db_config()
         git_largefiles_store = conf.get("vcs_git_lfs", "store_location")
 
         repo = backend_git.create_test_repo("largefiles", conf)
 
         tip = repo.scm_instance().get_commit()
         node = tip.get_node(b"1MB.zip")
 
         # extract stored LF node into the origin cache
         repo_lfs_store: str = os.path.join(repo.repo_path, repo.repo_name, "lfs_store")
 
         oid: str = "7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf"
         # where the OID actually is INSIDE the repo...
         oid_path = os.path.join(repo_lfs_store, oid)
 
         # Note: oid path depends on LFSOidStore.store_suffix. Once it will be changed update below line accordingly
         oid_destination = os.path.join(git_largefiles_store, f"objects/{oid[:2]}/{oid[2:4]}/{oid}")
 
         spec_path = os.path.dirname(oid_destination)
         os.makedirs(spec_path, exist_ok=True)
         shutil.copy(oid_path, oid_destination)
 
         lf_node = node.get_largefile_node()
 
         assert lf_node.is_largefile() is True
         assert lf_node.size == 1024000
         assert lf_node.name == b"1MB.zip"
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestGitSpecificWithRepo(BackendTestMixin):
     @classmethod
     def _get_commits(cls):
         return [
             {
                 "message": "Initial",
                 "author": "Joe Doe <[e-mail]>",
                 "date": datetime.datetime(2010, 1, 1, 20),
                 "added": [
                     FileNode(b"foobar/static/js/admin/base.js", content=b"base"),
                     FileNode(b"foobar/static/admin", content=b"admin", mode=0o120000),  # this is a link
                     FileNode(b"foo", content=b"foo"),
                 ],
             },
             {
                 "message": "Second",
                 "author": "Joe Doe <[e-mail]>",
                 "date": datetime.datetime(2010, 1, 1, 22),
                 "added": [
                     FileNode(b"foo2", content=b"foo2"),
                 ],
             },
         ]
 
     def test_paths_slow_traversing(self):
         commit = self.repo.get_commit()
         assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base"
 
     def test_paths_fast_traversing(self):
         commit = self.repo.get_commit()
         assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base"
 
     def test_get_diff_runs_git_command_with_hashes(self):
         comm1 = self.repo[0]
         comm2 = self.repo[1]
 
         with mock.patch.object(self.repo, "_remote", return_value=mock.Mock()) as remote_mock:
             remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff)
             self.repo.get_diff(comm1, comm2)
 
         remote_mock.diff.assert_called_once_with(
             comm1.raw_id, comm2.raw_id, file_filter=None, opt_ignorews=False, context=3
         )
 
     def test_get_diff_runs_git_command_with_str_hashes(self):
         comm2 = self.repo[1]
 
         with mock.patch.object(self.repo, "_remote", return_value=mock.Mock()) as remote_mock:
             remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff)
             self.repo.get_diff(self.repo.EMPTY_COMMIT, comm2)
 
         remote_mock.diff.assert_called_once_with(
             self.repo.EMPTY_COMMIT.raw_id, comm2.raw_id, file_filter=None, opt_ignorews=False, context=3
         )
 
     def test_get_diff_runs_git_command_with_path_if_its_given(self):
         comm1 = self.repo[0]
         comm2 = self.repo[1]
 
         with mock.patch.object(self.repo, "_remote", return_value=mock.Mock()) as remote_mock:
             remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff)
             self.repo.get_diff(comm1, comm2, "foo")
 
         remote_mock.diff.assert_called_once_with(
             self.repo._lookup_commit(0), comm2.raw_id, file_filter="foo", opt_ignorews=False, context=3
         )
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestGitRegression(BackendTestMixin):
     @classmethod
     def _get_commits(cls):
         return [
             {
                 "message": "Initial",
                 "author": "Joe Doe <[e-mail]>",
                 "date": datetime.datetime(2010, 1, 1, 20),
                 "added": [
                     FileNode(b"bot/__init__.py", content=b"base"),
                     FileNode(b"bot/templates/404.html", content=b"base"),
                     FileNode(b"bot/templates/500.html", content=b"base"),
                 ],
             },
             {
                 "message": "Second",
                 "author": "Joe Doe <[e-mail]>",
                 "date": datetime.datetime(2010, 1, 1, 22),
                 "added": [
                     FileNode(b"bot/build/migrations/1.py", content=b"foo2"),
                     FileNode(b"bot/build/migrations/2.py", content=b"foo2"),
                     FileNode(b"bot/build/static/templates/f.html", content=b"foo2"),
                     FileNode(b"bot/build/static/templates/f1.html", content=b"foo2"),
                     FileNode(b"bot/build/templates/err.html", content=b"foo2"),
                     FileNode(b"bot/build/templates/err2.html", content=b"foo2"),
                 ],
             },
         ]
 
     @pytest.mark.parametrize(
         "path, expected_paths",
         [
             (b"bot", ["bot/build", "bot/templates", "bot/__init__.py"]),
             (b"bot/build", ["bot/build/migrations", "bot/build/static", "bot/build/templates"]),
             (b"bot/build/static", ["bot/build/static/templates"]),
             (
                 b"bot/build/static/templates",
                 ["bot/build/static/templates/f.html", "bot/build/static/templates/f1.html"],
             ),
             (b"bot/build/templates", ["bot/build/templates/err.html", "bot/build/templates/err2.html"]),
             (b"bot/templates/", ["bot/templates/404.html", "bot/templates/500.html"]),
         ],
     )
     def test_similar_paths(self, path, expected_paths):
         commit = self.repo.get_commit()
         paths = [n.path for n in commit.get_nodes(path)]
         assert paths == expected_paths
 
 
 class TestDiscoverGitVersion(object):
     def test_returns_git_version(self, baseapp):
         version = discover_git_version()
         assert version
 
     def test_returns_empty_string_without_vcsserver(self):
         mock_connection = mock.Mock()
         mock_connection.discover_git_version = mock.Mock(side_effect=Exception)
         with mock.patch("rhodecode.lib.vcs.connection.Git", mock_connection):
             version = discover_git_version()
         assert version == ""
 
 
 class TestGetSubmoduleUrl(object):
     def test_submodules_file_found(self):
         commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
         node = mock.Mock()
 
         with mock.patch.object(commit, "get_node", return_value=node) as get_node_mock:
             node.str_content = (
                 '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
             )
             result = commit._get_submodule_url(b"subrepo1")
         get_node_mock.assert_called_once_with(b".gitmodules")
         assert result == "https://code.rhodecode.com/dulwich"
 
     def test_complex_submodule_path(self):
         commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
         node = mock.Mock()
 
         with mock.patch.object(commit, "get_node", return_value=node) as get_node_mock:
             node.str_content = (
                 '[submodule "complex/subrepo/path"]\n'
                 "\tpath = complex/subrepo/path\n"
                 "\turl = https://code.rhodecode.com/dulwich\n"
             )
             result = commit._get_submodule_url(b"complex/subrepo/path")
         get_node_mock.assert_called_once_with(b".gitmodules")
         assert result == "https://code.rhodecode.com/dulwich"
 
     def test_submodules_file_not_found(self):
         commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
         with mock.patch.object(commit, "get_node", side_effect=NodeDoesNotExistError):
             result = commit._get_submodule_url(b"complex/subrepo/path")
         assert result is None
 
     def test_path_not_found(self):
         commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
         node = mock.Mock()
 
         with mock.patch.object(commit, "get_node", return_value=node) as get_node_mock:
             node.str_content = (
                 '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
             )
             result = commit._get_submodule_url(b"subrepo2")
         get_node_mock.assert_called_once_with(b".gitmodules")
         assert result is None
 
     def test_returns_cached_values(self):
         commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
         node = mock.Mock()
 
         with mock.patch.object(commit, "get_node", return_value=node) as get_node_mock:
             node.str_content = (
                 '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
             )
             for _ in range(3):
                 commit._get_submodule_url(b"subrepo1")
         get_node_mock.assert_called_once_with(b".gitmodules")
 
     def test_get_node_returns_a_link(self):
         repository = mock.Mock()
         repository.alias = "git"
         commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1)
         submodule_url = "https://code.rhodecode.com/dulwich"
         get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.SUBMODULE))
         get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url)
 
         with get_id_patch, get_submodule_patch as submodule_mock:
             node = commit.get_node(b"/abcde")
 
         submodule_mock.assert_called_once_with(b"/abcde")
         assert type(node) == SubModuleNode
         assert node.url == submodule_url
 
     def test_get_nodes_returns_links(self):
         repository = mock.MagicMock()
         repository.alias = "git"
-        repository._remote.tree_items.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE)]
+        # obj_name, stat_, tree_item_id, node_kind, pre_load_data
+        repository._remote.get_nodes.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE, [])]
         commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1)
         submodule_url = "https://code.rhodecode.com/dulwich"
+
         get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.DIR))
         get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url)
 
         with get_id_patch, get_submodule_patch as submodule_mock:
             nodes = commit.get_nodes(b"/abcde")
 
-        submodule_mock.assert_called_once_with(b"/abcde/subrepo")
         assert len(nodes) == 1
         assert type(nodes[0]) == SubModuleNode
         assert nodes[0].url == submodule_url
+        submodule_mock.assert_called_once_with(b"/abcde/subrepo")
 
 
 class TestGetShadowInstance(object):
     @pytest.fixture()
     def repo(self, vcsbackend_git):
         _git_repo = vcsbackend_git.repo
 
         mock.patch.object(_git_repo, "config", mock.Mock())
         connection_mock = mock.Mock(unsafe=True, name="connection.Hg")
 
         mock.patch("rhodecode.lib.vcs.connection.Git", connection_mock)
         return _git_repo
 
     def test_getting_shadow_instance_copies_config(self, repo):
         shadow = repo.get_shadow_instance(repo.path)
         assert shadow.config.serialize() == repo.config.serialize()