# HG changeset patch # User RhodeCode Admin # Date 2024-12-04 21:33:45 # Node ID bad147da1438527a174c215fce2c974a3a056b37 # Parent cb066911dddf3dc3debda7e62d761a8d5470e832 fix(file-caching): fixed cases when old cache was used before changes to operate on bytestrings diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py --- a/rhodecode/apps/_base/__init__.py +++ b/rhodecode/apps/_base/__init__.py @@ -395,7 +395,7 @@ class RepoAppView(BaseAppView): settings = settings_model.get_repo_settings_inherited() return settings.get(settings_key, default) - def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path="/"): + def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path="/", nodes=None): log.debug("Looking for README file at path %s", path) if commit_id: landing_commit_id = commit_id @@ -413,16 +413,14 @@ class RepoAppView(BaseAppView): @region.conditional_cache_on_arguments(namespace=cache_namespace_uid) def generate_repo_readme( - repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type + _repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type ): - readme_data = None - readme_filename = None + _readme_data = None + _readme_filename = None commit = db_repo.get_commit(_commit_id) log.debug("Searching for a README file at commit %s.", _commit_id) - readme_node = ReadmeFinder(_renderer_type).search( - commit, path=_readme_search_path - ) + readme_node = ReadmeFinder(_renderer_type).search(commit, path=_readme_search_path, nodes=nodes) if readme_node: log.debug("Found README node: %s", readme_node) @@ -442,19 +440,19 @@ class RepoAppView(BaseAppView): ), } - readme_data = self._render_readme_or_none( + _readme_data = self._render_readme_or_none( commit, readme_node, relative_urls ) - readme_filename = readme_node.str_path + _readme_filename = readme_node.str_path - return readme_data, readme_filename + return _readme_data, _readme_filename readme_data, readme_filename = generate_repo_readme( db_repo.repo_id, landing_commit_id, db_repo.repo_name, path, - renderer_type, + renderer_type ) compute_time = time.time() - start diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -40,25 +40,34 @@ from rhodecode.lib import diffs, helpers from rhodecode.lib import audit_logger from rhodecode.lib.hash_utils import sha1_safe from rhodecode.lib.archive_cache import ( - get_archival_cache_store, get_archival_config, ArchiveCacheGenerationLock, archive_iterator) + get_archival_cache_store, + get_archival_config, + ArchiveCacheGenerationLock, + archive_iterator, +) from rhodecode.lib.str_utils import safe_bytes, convert_special_chars from rhodecode.lib.view_utils import parse_path_ref from rhodecode.lib.exceptions import NonRelativePathError -from rhodecode.lib.codeblocks import ( - filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) +from rhodecode.lib.codeblocks import filenode_as_lines_tokens, filenode_as_annotated_lines_tokens from rhodecode.lib.utils2 import convert_line_endings, detect_mode from rhodecode.lib.type_utils import str2bool from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str -from rhodecode.lib.auth import ( - LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) +from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired from rhodecode.lib.vcs import path as vcspath from rhodecode.lib.vcs.backends.base import EmptyCommit from rhodecode.lib.vcs.conf import settings from rhodecode.lib.vcs.nodes import FileNode from rhodecode.lib.vcs.exceptions import ( - RepositoryError, CommitDoesNotExistError, EmptyRepositoryError, - ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError, - NodeDoesNotExistError, CommitError, NodeError) + RepositoryError, + CommitDoesNotExistError, + EmptyRepositoryError, + ImproperArchiveTypeError, + VCSError, + NodeAlreadyExistsError, + NodeDoesNotExistError, + CommitError, + NodeError, +) from rhodecode.model.scm import ScmModel from rhodecode.model.db import Repository @@ -66,17 +75,17 @@ from rhodecode.model.db import Repositor log = logging.getLogger(__name__) -def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True): +def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha="", with_hash=True): # original backward compat name of archive - clean_name = safe_str(convert_special_chars(db_repo_name).replace('/', '_')) + clean_name = safe_str(convert_special_chars(db_repo_name).replace("/", "_")) # e.g vcsserver-id-abcd-sub-1-abcfdef-archive-all.zip # vcsserver-id-abcd-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip id_sha = sha1_safe(str(db_repo_id))[:4] - sub_repo = 'sub-1' if subrepos else 'sub-0' - commit = commit_sha if with_hash else 'archive' - path_marker = (path_sha if with_hash else '') or 'all' - archive_name = f'{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}' + sub_repo = "sub-1" if subrepos else "sub-0" + commit = commit_sha if with_hash else "archive" + path_marker = (path_sha if with_hash else "") or "all" + archive_name = f"{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}" return archive_name @@ -86,16 +95,15 @@ def get_path_sha(at_path): def _get_archive_spec(fname): - log.debug('Detecting archive spec for: `%s`', fname) + log.debug("Detecting archive spec for: `%s`", fname) fileformat = None ext = None content_type = None for a_type, content_type, extension in settings.ARCHIVE_SPECS: - if fname.endswith(extension): fileformat = a_type - log.debug('archive is of type: %s', fileformat) + log.debug("archive is of type: %s", fileformat) ext = extension break @@ -103,13 +111,12 @@ def _get_archive_spec(fname): raise ValueError() # left over part of whole fname is the commit - commit_id = fname[:-len(ext)] + commit_id = fname[: -len(ext)] return commit_id, ext, fileformat, content_type class RepoFilesView(RepoAppView): - @staticmethod def adjust_file_path_for_svn(f_path, repo): """ @@ -118,13 +125,11 @@ class RepoFilesView(RepoAppView): This is mainly based on prefix matching of the recognized tags and branches in the underlying repository. """ - tags_and_branches = itertools.chain( - repo.branches.keys(), - repo.tags.keys()) + tags_and_branches = itertools.chain(repo.branches.keys(), repo.tags.keys()) tags_and_branches = sorted(tags_and_branches, key=len, reverse=True) for name in tags_and_branches: - if f_path.startswith(f'{name}/'): + if f_path.startswith(f"{name}/"): f_path = vcspath.relpath(f_path, name) break return f_path @@ -135,59 +140,52 @@ class RepoFilesView(RepoAppView): c.enable_downloads = self.db_repo.enable_downloads return c - def _ensure_not_locked(self, commit_id='tip'): + def _ensure_not_locked(self, commit_id="tip"): _ = self.request.translate repo = self.db_repo if repo.enable_locking and repo.locked[0]: - h.flash(_('This repository has been locked by %s on %s') - % (h.person_by_id(repo.locked[0]), - h.format_date(h.time_to_datetime(repo.locked[1]))), - 'warning') - files_url = h.route_path( - 'repo_files:default_path', - repo_name=self.db_repo_name, commit_id=commit_id) + h.flash( + _("This repository has been locked by %s on %s") + % (h.person_by_id(repo.locked[0]), h.format_date(h.time_to_datetime(repo.locked[1]))), + "warning", + ) + files_url = h.route_path("repo_files:default_path", repo_name=self.db_repo_name, commit_id=commit_id) raise HTTPFound(files_url) - def forbid_non_head(self, is_head, f_path, commit_id='tip', json_mode=False): + def forbid_non_head(self, is_head, f_path, commit_id="tip", json_mode=False): _ = self.request.translate if not is_head: - message = _('Cannot modify file. ' - 'Given commit `{}` is not head of a branch.').format(commit_id) - h.flash(message, category='warning') + message = _("Cannot modify file. " "Given commit `{}` is not head of a branch.").format(commit_id) + h.flash(message, category="warning") if json_mode: return message - files_url = h.route_path( - 'repo_files', repo_name=self.db_repo_name, commit_id=commit_id, - f_path=f_path) + files_url = h.route_path("repo_files", repo_name=self.db_repo_name, commit_id=commit_id, f_path=f_path) raise HTTPFound(files_url) - def check_branch_permission(self, branch_name, commit_id='tip', json_mode=False): + def check_branch_permission(self, branch_name, commit_id="tip", json_mode=False): _ = self.request.translate - rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission( - self.db_repo_name, branch_name) - if branch_perm and branch_perm not in ['branch.push', 'branch.push_force']: - message = _('Branch `{}` changes forbidden by rule {}.').format( - h.escape(branch_name), h.escape(rule)) - h.flash(message, 'warning') + rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission(self.db_repo_name, branch_name) + if branch_perm and branch_perm not in ["branch.push", "branch.push_force"]: + message = _("Branch `{}` changes forbidden by rule {}.").format(h.escape(branch_name), h.escape(rule)) + h.flash(message, "warning") if json_mode: return message - files_url = h.route_path( - 'repo_files:default_path', repo_name=self.db_repo_name, commit_id=commit_id) + files_url = h.route_path("repo_files:default_path", repo_name=self.db_repo_name, commit_id=commit_id) raise HTTPFound(files_url) def _get_commit_and_path(self): default_commit_id = self.db_repo.landing_ref_name - default_f_path = '/' + default_f_path = "/" - commit_id = self.request.matchdict.get('commit_id', default_commit_id) + commit_id = self.request.matchdict.get("commit_id", default_commit_id) f_path = self._get_f_path(self.request.matchdict, default_f_path) bytes_path = safe_bytes(f_path) @@ -195,8 +193,8 @@ class RepoFilesView(RepoAppView): @classmethod def _get_default_encoding(cls, c): - enc_list = getattr(c, 'default_encodings', []) - return enc_list[0] if enc_list else 'UTF-8' + enc_list = getattr(c, "default_encodings", []) + return enc_list[0] if enc_list else "UTF-8" def _get_commit_or_redirect(self, commit_id, redirect_after=True): """ @@ -215,31 +213,25 @@ class RepoFilesView(RepoAppView): return None add_new = upload_new = "" - if h.HasRepoPermissionAny( - 'repository.write', 'repository.admin')(self.db_repo_name): - _url = h.route_path( - 'repo_files_add_file', - repo_name=self.db_repo_name, commit_id=0, f_path='') - add_new = h.link_to( - _('add a new file'), _url, class_="alert-link") + if h.HasRepoPermissionAny("repository.write", "repository.admin")(self.db_repo_name): + _url = h.route_path("repo_files_add_file", repo_name=self.db_repo_name, commit_id=0, f_path="") + add_new = h.link_to(_("add a new file"), _url, class_="alert-link") - _url_upld = h.route_path( - 'repo_files_upload_file', - repo_name=self.db_repo_name, commit_id=0, f_path='') - upload_new = h.link_to( - _('upload a new file'), _url_upld, class_="alert-link") + _url_upld = h.route_path("repo_files_upload_file", repo_name=self.db_repo_name, commit_id=0, f_path="") + upload_new = h.link_to(_("upload a new file"), _url_upld, class_="alert-link") - h.flash(h.literal( - _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning') - raise HTTPFound( - h.route_path('repo_summary', repo_name=self.db_repo_name)) + h.flash( + h.literal(_("There are no files yet. Click here to %s or %s.") % (add_new, upload_new)), + category="warning", + ) + raise HTTPFound(h.route_path("repo_summary", repo_name=self.db_repo_name)) except (CommitDoesNotExistError, LookupError) as e: - msg = _('No such commit exists for this repository. Commit: {}').format(commit_id) - h.flash(msg, category='error') + msg = _("No such commit exists for this repository. Commit: {}").format(commit_id) + h.flash(msg, category="error") raise HTTPNotFound() except RepositoryError as e: - h.flash(h.escape(safe_str(e)), category='error') + h.flash(h.escape(safe_str(e)), category="error") raise HTTPNotFound() def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None): @@ -252,22 +244,22 @@ class RepoFilesView(RepoAppView): try: file_node = commit_obj.get_node(path, pre_load=pre_load) if file_node.is_dir(): - raise RepositoryError('The given path is a directory') + raise RepositoryError("The given path is a directory") except CommitDoesNotExistError: - log.exception('No such commit exists for this repository') - h.flash(_('No such commit exists for this repository'), category='error') + log.exception("No such commit exists for this repository") + h.flash(_("No such commit exists for this repository"), category="error") raise HTTPNotFound() except RepositoryError as e: - log.warning('Repository error while fetching filenode `%s`. Err:%s', path, e) - h.flash(h.escape(safe_str(e)), category='error') + log.warning("Repository error while fetching filenode `%s`. Err:%s", path, e) + h.flash(h.escape(safe_str(e)), category="error") raise HTTPNotFound() return file_node def _is_valid_head(self, commit_id, repo, landing_ref): - branch_name = sha_commit_id = '' + branch_name = sha_commit_id = "" is_head = False - log.debug('Checking if commit_id `%s` is a head for %s.', commit_id, repo) + log.debug("Checking if commit_id `%s` is a head for %s.", commit_id, repo) for _branch_name, branch_commit_id in repo.branches.items(): # simple case we pass in branch name, it's a HEAD @@ -303,39 +295,39 @@ class RepoFilesView(RepoAppView): return branch_name, sha_commit_id, is_head def _get_tree_at_commit(self, c, commit_id, f_path, full_load=False, at_rev=None): - repo_id = self.db_repo.repo_id force_recache = self.get_recache_flag() - cache_seconds = safe_int( - rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) + cache_seconds = rhodecode.ConfigGet().get_int("rc_cache.cache_repo.expiration_time") cache_on = not force_recache and cache_seconds > 0 + log.debug( - 'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`' - 'with caching: %s[TTL: %ss]' % ( - repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) + "Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`" + "with caching: %s[TTL: %ss]" % (repo_id, commit_id, f_path, cache_on, cache_seconds or 0) + ) - cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}' - region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + cache_namespace_uid = f"repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}" + region = rc_cache.get_or_create_region("cache_repo", cache_namespace_uid) @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev): - log.debug('Generating cached file tree at for repo_id: %s, %s, %s', - _repo_id, _commit_id, _f_path) + log.debug("Generating cached file tree at for repo_id: %s, %s, %s", _repo_id, _commit_id, _f_path) c.full_load = _full_load return render( - 'rhodecode:templates/files/files_browser_tree.mako', - self._get_template_context(c), self.request, _at_rev) + "rhodecode:templates/files/files_browser_tree.mako", + self._get_template_context(c), + self.request, + _at_rev, + ) return compute_file_tree( - self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev) + self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev + ) def create_pure_path(self, *parts): # Split paths and sanitize them, removing any ../ etc - sanitized_path = [ - x for x in pathlib.PurePath(*parts).parts - if x not in ['.', '..']] + sanitized_path = [x for x in pathlib.PurePath(*parts).parts if x not in [".", ".."]] pure_path = pathlib.PurePath(*sanitized_path) return pure_path @@ -343,10 +335,7 @@ class RepoFilesView(RepoAppView): def _is_lf_enabled(self, target_repo): lf_enabled = False - lf_key_for_vcs_map = { - 'hg': 'extensions_largefiles', - 'git': 'vcs_git_lfs_enabled' - } + lf_key_for_vcs_map = {"hg": "extensions_largefiles", "git": "vcs_git_lfs_enabled"} lf_key_for_vcs = lf_key_for_vcs_map.get(target_repo.repo_type) @@ -356,66 +345,77 @@ class RepoFilesView(RepoAppView): return lf_enabled @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_archivefile(self): # archive cache config from rhodecode import CONFIG + _ = self.request.translate self.load_default_context() - subrepos = self.request.GET.get('subrepos') == 'true' - with_hash = str2bool(self.request.GET.get('with_hash', '1')) + subrepos = self.request.GET.get("subrepos") == "true" + with_hash = str2bool(self.request.GET.get("with_hash", "1")) - default_at_path = '/' - fname = self.request.matchdict['fname'] - at_path = self.request.GET.get('at_path') or default_at_path + default_at_path = "/" + fname = self.request.matchdict["fname"] + at_path = self.request.GET.get("at_path") or default_at_path if not self.db_repo.enable_downloads: - return Response(_('Downloads disabled')) + return Response(_("Downloads disabled")) try: commit_id, ext, file_format, content_type = _get_archive_spec(fname) except ValueError: - return Response(_('Unknown archive type for: `{}`').format(h.escape(fname))) + return Response(_("Unknown archive type for: `{}`").format(h.escape(fname))) try: commit = self.rhodecode_vcs_repo.get_commit(commit_id) except CommitDoesNotExistError: - return Response(_('Unknown commit_id {}').format( - h.escape(commit_id))) + return Response(_("Unknown commit_id {}").format(h.escape(commit_id))) except EmptyRepositoryError: - return Response(_('Empty repository')) + return Response(_("Empty repository")) # we used a ref, or a shorter version, lets redirect client ot use explicit hash if commit_id != commit.raw_id: - fname=f'{commit.raw_id}{ext}' + fname = f"{commit.raw_id}{ext}" raise HTTPFound(self.request.current_route_path(fname=fname)) try: at_path = commit.get_node(safe_bytes(at_path)).path or default_at_path except Exception: - return Response(_('No node at path {} for this repository').format(h.escape(at_path))) + return Response(_("No node at path {} for this repository").format(h.escape(at_path))) path_sha = get_path_sha(at_path) # used for cache etc, consistent unique archive name archive_name_key = get_archive_name( - self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, - path_sha=path_sha, with_hash=True) + self.db_repo.repo_id, + self.db_repo_name, + commit_sha=commit.short_id, + ext=ext, + subrepos=subrepos, + path_sha=path_sha, + with_hash=True, + ) if not with_hash: - path_sha = '' + path_sha = "" # what end client gets served response_archive_name = get_archive_name( - self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, - path_sha=path_sha, with_hash=with_hash) + self.db_repo.repo_id, + self.db_repo_name, + commit_sha=commit.short_id, + ext=ext, + subrepos=subrepos, + path_sha=path_sha, + with_hash=with_hash, + ) # remove extension from our archive directory name - archive_dir_name = response_archive_name[:-len(ext)] + archive_dir_name = response_archive_name[: -len(ext)] - archive_cache_disable = self.request.GET.get('no_cache') + archive_cache_disable = self.request.GET.get("no_cache") d_cache = get_archival_cache_store(config=CONFIG) @@ -423,29 +423,38 @@ class RepoFilesView(RepoAppView): d_cache_conf = get_archival_config(config=CONFIG) # This is also a cache key, and lock key - reentrant_lock_key = archive_name_key + '.lock' + reentrant_lock_key = archive_name_key + ".lock" use_cached_archive = False if not archive_cache_disable and archive_name_key in d_cache: reader, metadata = d_cache.fetch(archive_name_key) use_cached_archive = True - log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s', - archive_name_key, metadata, reader.name) + log.debug( + "Found cached archive as key=%s tag=%s, serving archive from cache reader=%s", + archive_name_key, + metadata, + reader.name, + ) else: reader = None - log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key) + log.debug("Archive with key=%s is not yet cached, creating one now...", archive_name_key) if not reader: # generate new archive, as previous was not found in the cache try: with d_cache.get_lock(reentrant_lock_key): try: - commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name, - kind=file_format, subrepos=subrepos, - archive_at_path=at_path, cache_config=d_cache_conf) + commit.archive_repo( + archive_name_key, + archive_dir_name=archive_dir_name, + kind=file_format, + subrepos=subrepos, + archive_at_path=at_path, + cache_config=d_cache_conf, + ) except ImproperArchiveTypeError: - return _('Unknown archive type') + return _("Unknown archive type") except ArchiveCacheGenerationLock: retry_after = round(random.uniform(0.3, 3.0), 1) @@ -464,7 +473,7 @@ class RepoFilesView(RepoAppView): reader, metadata = d_cache.fetch(archive_name_key, retry=True, retry_attempts=30) response = Response(app_iter=archive_iterator(reader)) - response.content_disposition = f'attachment; filename={response_archive_name}' + response.content_disposition = f"attachment; filename={response_archive_name}" response.content_type = str(content_type) try: @@ -472,23 +481,25 @@ class RepoFilesView(RepoAppView): finally: # store download action audit_logger.store_web( - 'repo.archive.download', action_data={ - 'user_agent': self.request.user_agent, - 'archive_name': archive_name_key, - 'archive_spec': fname, - 'archive_cached': use_cached_archive}, + "repo.archive.download", + action_data={ + "user_agent": self.request.user_agent, + "archive_name": archive_name_key, + "archive_spec": fname, + "archive_cached": use_cached_archive, + }, user=self._rhodecode_user, repo=self.db_repo, - commit=True + commit=True, ) def _get_file_node(self, commit_id, f_path): - if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]: + if commit_id not in ["", None, "None", "0" * 12, "0" * 40]: commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id) try: node = commit.get_node(safe_bytes(f_path)) if node.is_dir(): - raise NodeError(f'{node} path is a {type(node)} not a file') + raise NodeError(f"{node} path is a {type(node)} not a file") except NodeDoesNotExistError: commit = EmptyCommit( commit_id=commit_id, @@ -497,46 +508,43 @@ class RepoFilesView(RepoAppView): alias=commit.repository.alias, message=commit.message, author=commit.author, - date=commit.date) - node = FileNode(safe_bytes(f_path), b'', commit=commit) + date=commit.date, + ) + node = FileNode(safe_bytes(f_path), b"", commit=commit) else: - commit = EmptyCommit( - repo=self.rhodecode_vcs_repo, - alias=self.rhodecode_vcs_repo.alias) - node = FileNode(safe_bytes(f_path), b'', commit=commit) + commit = EmptyCommit(repo=self.rhodecode_vcs_repo, alias=self.rhodecode_vcs_repo.alias) + node = FileNode(safe_bytes(f_path), b"", commit=commit) return node @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_files_diff(self): c = self.load_default_context() f_path = self._get_f_path(self.request.matchdict) - diff1 = self.request.GET.get('diff1', '') - diff2 = self.request.GET.get('diff2', '') + diff1 = self.request.GET.get("diff1", "") + diff2 = self.request.GET.get("diff2", "") path1, diff1 = parse_path_ref(diff1, default_path=f_path) - ignore_whitespace = str2bool(self.request.GET.get('ignorews')) - line_context = self.request.GET.get('context', 3) + ignore_whitespace = str2bool(self.request.GET.get("ignorews")) + line_context = self.request.GET.get("context", 3) if not any((diff1, diff2)): - h.flash( - 'Need query parameter "diff1" or "diff2" to generate a diff.', - category='error') + h.flash('Need query parameter "diff1" or "diff2" to generate a diff.', category="error") raise HTTPBadRequest() - c.action = self.request.GET.get('diff') - if c.action not in ['download', 'raw']: + c.action = self.request.GET.get("diff") + if c.action not in ["download", "raw"]: compare_url = h.route_path( - 'repo_compare', + "repo_compare", repo_name=self.db_repo_name, - source_ref_type='rev', + source_ref_type="rev", source_ref=diff1, target_repo=self.db_repo_name, - target_ref_type='rev', + target_ref_type="rev", target_ref=diff2, - _query=dict(f_path=f_path)) + _query=dict(f_path=f_path), + ) # redirect to new view if we render diff raise HTTPFound(compare_url) @@ -545,43 +553,34 @@ class RepoFilesView(RepoAppView): node2 = self._get_file_node(diff2, f_path) except (RepositoryError, NodeError): log.exception("Exception while trying to get node from repository") - raise HTTPFound( - h.route_path('repo_files', repo_name=self.db_repo_name, - commit_id='tip', f_path=f_path)) + raise HTTPFound(h.route_path("repo_files", repo_name=self.db_repo_name, commit_id="tip", f_path=f_path)) - if all(isinstance(node.commit, EmptyCommit) - for node in (node1, node2)): + if all(isinstance(node.commit, EmptyCommit) for node in (node1, node2)): raise HTTPNotFound() c.commit_1 = node1.commit c.commit_2 = node2.commit - if c.action == 'download': - _diff = diffs.get_gitdiff(node1, node2, - ignore_whitespace=ignore_whitespace, - context=line_context) + if c.action == "download": + _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context) # NOTE: this was using diff_format='gitdiff' - diff = diffs.DiffProcessor(_diff, diff_format='newdiff') + diff = diffs.DiffProcessor(_diff, diff_format="newdiff") response = Response(self.path_filter.get_raw_patch(diff)) - response.content_type = 'text/plain' - response.content_disposition = ( - f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff' - ) + response.content_type = "text/plain" + response.content_disposition = f"attachment; filename={f_path}_{diff1}_vs_{diff2}.diff" charset = self._get_default_encoding(c) if charset: response.charset = charset return response - elif c.action == 'raw': - _diff = diffs.get_gitdiff(node1, node2, - ignore_whitespace=ignore_whitespace, - context=line_context) + elif c.action == "raw": + _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context) # NOTE: this was using diff_format='gitdiff' - diff = diffs.DiffProcessor(_diff, diff_format='newdiff') + diff = diffs.DiffProcessor(_diff, diff_format="newdiff") response = Response(self.path_filter.get_raw_patch(diff)) - response.content_type = 'text/plain' + response.content_type = "text/plain" charset = self._get_default_encoding(c) if charset: response.charset = charset @@ -591,31 +590,32 @@ class RepoFilesView(RepoAppView): raise HTTPNotFound() @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_files_diff_2way_redirect(self): """ Kept only to make OLD links work """ f_path = self._get_f_path_unchecked(self.request.matchdict) - diff1 = self.request.GET.get('diff1', '') - diff2 = self.request.GET.get('diff2', '') + diff1 = self.request.GET.get("diff1", "") + diff2 = self.request.GET.get("diff2", "") if not any((diff1, diff2)): - h.flash( - 'Need query parameter "diff1" or "diff2" to generate a diff.', - category='error') + h.flash('Need query parameter "diff1" or "diff2" to generate a diff.', category="error") raise HTTPBadRequest() compare_url = h.route_path( - 'repo_compare', + "repo_compare", repo_name=self.db_repo_name, - source_ref_type='rev', + source_ref_type="rev", source_ref=diff1, - target_ref_type='rev', + target_ref_type="rev", target_ref=diff2, - _query=dict(f_path=f_path, diffmode='sideside', - target_repo=self.db_repo_name,)) + _query=dict( + f_path=f_path, + diffmode="sideside", + target_repo=self.db_repo_name, + ), + ) raise HTTPFound(compare_url) @LoginRequired() @@ -629,138 +629,131 @@ class RepoFilesView(RepoAppView): landing_url = h.repo_files_by_ref_url( c.rhodecode_db_repo.repo_name, c.rhodecode_db_repo.repo_type, - f_path='', + f_path="", ref_name=ref_name, - commit_id='tip', - query=dict(at=ref_name) + commit_id="tip", + query=dict(at=ref_name), ) raise HTTPFound(landing_url) @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_files(self): c = self.load_default_context() - view_name = getattr(self.request.matched_route, 'name', None) + view_name = getattr(self.request.matched_route, "name", None) - c.annotate = view_name == 'repo_files:annotated' + c.annotate = view_name == "repo_files:annotated" # default is false, but .rst/.md files later are auto rendered, we can # overwrite auto rendering by setting this GET flag - c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False) + c.renderer = view_name == "repo_files:rendered" or not self.request.GET.get("no-render", False) commit_id, f_path, bytes_path = self._get_commit_and_path() c.commit = self._get_commit_or_redirect(commit_id) - c.branch = self.request.GET.get('branch', None) + c.branch = self.request.GET.get("branch", None) c.f_path = f_path - at_rev = self.request.GET.get('at') + at_rev = self.request.GET.get("at") # files or dirs try: - - c.file = c.commit.get_node(bytes_path, pre_load=['is_binary', 'size', 'data']) + c.file = c.commit.get_node(bytes_path, pre_load=["is_binary", "size", "data"]) c.file_author = True - c.file_tree = '' + c.file_tree = "" # prev link try: prev_commit = c.commit.prev(c.branch) c.prev_commit = prev_commit - c.url_prev = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path) + c.url_prev = h.route_path( + "repo_files", repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path + ) if c.branch: - c.url_prev += f'?branch={c.branch}' + c.url_prev += f"?branch={c.branch}" except (CommitDoesNotExistError, VCSError): - c.url_prev = '#' + c.url_prev = "#" c.prev_commit = EmptyCommit() # next link try: next_commit = c.commit.next(c.branch) c.next_commit = next_commit - c.url_next = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path) + c.url_next = h.route_path( + "repo_files", repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path + ) if c.branch: - c.url_next += f'?branch={c.branch}' + c.url_next += f"?branch={c.branch}" except (CommitDoesNotExistError, VCSError): - c.url_next = '#' + c.url_next = "#" c.next_commit = EmptyCommit() # load file content if c.file.is_file(): - c.lf_node = {} has_lf_enabled = self._is_lf_enabled(self.db_repo) if has_lf_enabled: c.lf_node = c.file.get_largefile_node() - c.file_source_page = 'true' + c.file_source_page = "true" c.file_last_commit = c.file.last_commit c.file_size_too_big = c.file.size > c.visual.cut_off_limit_file if not (c.file_size_too_big or c.file.is_binary): if c.annotate: # annotation has precedence over renderer - c.annotated_lines = filenode_as_annotated_lines_tokens( - c.file - ) + c.annotated_lines = filenode_as_annotated_lines_tokens(c.file) else: - c.renderer = ( - c.renderer and h.renderer_from_filename(c.file.path) - ) + c.renderer = c.renderer and h.renderer_from_filename(c.file.path) if not c.renderer: c.lines = filenode_as_lines_tokens(c.file) - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) c.on_branch_head = is_head - branch = c.commit.branch if ( - c.commit.branch and '/' not in c.commit.branch) else None + branch = c.commit.branch if (c.commit.branch and "/" not in c.commit.branch) else None c.branch_or_raw_id = branch or c.commit.raw_id c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id) author = c.file_last_commit.author - c.authors = [[ - h.email(author), - h.person(author, 'username_or_name_or_email'), - 1 - ]] + c.authors = [[h.email(author), h.person(author, "username_or_name_or_email"), 1]] - else: # load tree content at path - c.file_source_page = 'false' + else: # load tree content (dir content) at path + c.file_source_page = "false" c.authors = [] + + dir_node = c.file + c.file_nodes = dir_node.commit.get_nodes(dir_node.bytes_path, pre_load=dir_node.default_pre_load) # this loads a simple tree without metadata to speed things up # later via ajax we call repo_nodetree_full and fetch whole c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev) - c.readme_data, c.readme_file = \ - self._get_readme_data(self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path) + c.readme_data, c.readme_file = self._get_readme_data( + self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path, nodes=c.file_nodes + ) except RepositoryError as e: - h.flash(h.escape(safe_str(e)), category='error') + h.flash(h.escape(safe_str(e)), category="error") raise HTTPNotFound() - if self.request.environ.get('HTTP_X_PJAX'): - html = render('rhodecode:templates/files/files_pjax.mako', - self._get_template_context(c), self.request) + if self.request.environ.get("HTTP_X_PJAX"): + html = render("rhodecode:templates/files/files_pjax.mako", self._get_template_context(c), self.request) else: - html = render('rhodecode:templates/files/files.mako', - self._get_template_context(c), self.request) + html = render("rhodecode:templates/files/files.mako", self._get_template_context(c), self.request) return Response(html) - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_files_annotated_previous(self): self.load_default_context() commit_id, bytes_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) prev_commit_id = commit.raw_id - line_anchor = self.request.GET.get('line_anchor') + line_anchor = self.request.GET.get("line_anchor") is_file = False try: _file = commit.get_node(bytes_path) @@ -770,18 +763,19 @@ class RepoFilesView(RepoAppView): if is_file: history = commit.get_path_history(bytes_path) - prev_commit_id = history[1].raw_id \ - if len(history) > 1 else prev_commit_id + prev_commit_id = history[1].raw_id if len(history) > 1 else prev_commit_id prev_url = h.route_path( - 'repo_files:annotated', repo_name=self.db_repo_name, - commit_id=prev_commit_id, f_path=bytes_path, - _anchor=f'L{line_anchor}') + "repo_files:annotated", + repo_name=self.db_repo_name, + commit_id=prev_commit_id, + f_path=bytes_path, + _anchor=f"L{line_anchor}", + ) raise HTTPFound(prev_url) @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_nodetree_full(self): """ Returns rendered html of file tree that contains commit date, @@ -795,17 +789,17 @@ class RepoFilesView(RepoAppView): try: dir_node = commit.get_node(bytes_path) except RepositoryError as e: - return Response(f'error: {h.escape(safe_str(e))}') + return Response(f"error: {h.escape(safe_str(e))}") if dir_node.is_file(): - return Response('') + return Response("") c.file = dir_node + c.file_nodes = dir_node.commit.get_nodes(dir_node.bytes_path, pre_load=dir_node.default_pre_load) c.commit = commit - at_rev = self.request.GET.get('at') + at_rev = self.request.GET.get("at") - html = self._get_tree_at_commit( - c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev) + html = self._get_tree_at_commit(c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev) return Response(html) @@ -814,15 +808,12 @@ class RepoFilesView(RepoAppView): safe_path = f_name.replace('"', '\\"') encoded_path = urllib.parse.quote(f_name) - headers = f"attachment; " \ - f"filename=\"{safe_path}\"; " \ - f"filename*=UTF-8\'\'{encoded_path}" + headers = f"attachment; " f'filename="{safe_path}"; ' f"filename*=UTF-8''{encoded_path}" return header_safe_str(headers) @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_file_raw(self): """ Action for show as raw, some mimetypes are "rendered", @@ -839,16 +830,15 @@ class RepoFilesView(RepoAppView): # you can also provide a content-disposition to override the # default "attachment" disposition. # orig_type: (new_type, new_dispo) - # show images inline: # Do not re-add SVG: it is unsafe and permits XSS attacks. One can # for example render an SVG with javascript inside or even render # HTML. - 'image/x-icon': ('image/x-icon', 'inline'), - 'image/png': ('image/png', 'inline'), - 'image/gif': ('image/gif', 'inline'), - 'image/jpeg': ('image/jpeg', 'inline'), - 'application/pdf': ('application/pdf', 'inline'), + "image/x-icon": ("image/x-icon", "inline"), + "image/png": ("image/png", "inline"), + "image/gif": ("image/gif", "inline"), + "image/jpeg": ("image/jpeg", "inline"), + "application/pdf": ("application/pdf", "inline"), } mimetype = file_node.mimetype @@ -858,7 +848,7 @@ class RepoFilesView(RepoAppView): # we don't know anything special about this, handle it safely if file_node.is_binary: # do same as download raw for binary files - mimetype, disposition = 'application/octet-stream', 'attachment' + mimetype, disposition = "application/octet-stream", "attachment" else: # do not just use the original mimetype, but force text/plain, # otherwise it would serve text/html and that might be unsafe. @@ -867,9 +857,9 @@ class RepoFilesView(RepoAppView): # binary.This might lead to erroneous text display in some # cases, but helps in other cases, like with text files # without extension. - mimetype, disposition = 'text/plain', 'inline' + mimetype, disposition = "text/plain", "inline" - if disposition == 'attachment': + if disposition == "attachment": disposition = self._get_attachement_headers(f_path) stream_content = file_node.stream_bytes() @@ -885,8 +875,7 @@ class RepoFilesView(RepoAppView): return response @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_file_download(self): c = self.load_default_context() @@ -894,7 +883,7 @@ class RepoFilesView(RepoAppView): commit = self._get_commit_or_redirect(commit_id) file_node = self._get_filenode_or_redirect(commit, bytes_path) - if self.request.GET.get('lf'): + if self.request.GET.get("lf"): # only if lf get flag is passed, we download this file # as LFS/Largefile lf_node = file_node.get_largefile_node() @@ -917,44 +906,41 @@ class RepoFilesView(RepoAppView): return response def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path): - - cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time') + cache_seconds = rhodecode.ConfigGet().get_int("rc_cache.cache_repo.expiration_time") cache_on = cache_seconds > 0 log.debug( - 'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`' - 'with caching: %s[TTL: %ss]' % ( - repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) + "Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`" + "with caching: %s[TTL: %ss]" % (repo_id, commit_id, f_path, cache_on, cache_seconds or 0) + ) - cache_namespace_uid = f'repo.{repo_id}' - region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) + cache_namespace_uid = f"repo.{repo_id}" + region = rc_cache.get_or_create_region("cache_repo", cache_namespace_uid) @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path): - log.debug('Generating cached nodelist for repo_id:%s, %s, %s', _repo_id, commit_id, f_path) + log.debug("Generating cached nodelist for repo_id:%s, %s, %s", _repo_id, commit_id, f_path) try: _d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path) except (RepositoryError, CommitDoesNotExistError, Exception) as e: log.exception(safe_str(e)) - h.flash(h.escape(safe_str(e)), category='error') - raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, commit_id='tip', f_path='/')) + h.flash(h.escape(safe_str(e)), category="error") + raise HTTPFound(h.route_path("repo_files", repo_name=self.db_repo_name, commit_id="tip", f_path="/")) return _d + _f result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path) - return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result) + return filter(lambda n: self.path_filter.path_access_allowed(n["name"]), result) @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_nodelist(self): self.load_default_context() commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) - metadata = self._get_nodelist_at_commit( - self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path) - return {'nodes': [x for x in metadata]} + metadata = self._get_nodelist_at_commit(self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path) + return {"nodes": [x for x in metadata]} def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type): items = [] @@ -971,7 +957,7 @@ class RepoFilesView(RepoAppView): # NOTE(dan): old code we used in "diff" mode compare new_f_path = vcspath.join(name, f_path) - return f'{new_f_path}@{commit_id}' + return f"{new_f_path}@{commit_id}" def _get_node_history(self, commit_obj, f_path, commits=None): """ @@ -997,29 +983,26 @@ class RepoFilesView(RepoAppView): history = [] commits_group = ([], _("Changesets")) for commit in commits: - branch = ' (%s)' % commit.branch if commit.branch else '' - n_desc = f'r{commit.idx}:{commit.short_id}{branch}' - commits_group[0].append((commit.raw_id, n_desc, 'sha')) + branch = " (%s)" % commit.branch if commit.branch else "" + n_desc = f"r{commit.idx}:{commit.short_id}{branch}" + commits_group[0].append((commit.raw_id, n_desc, "sha")) history.append(commits_group) symbolic_reference = self._symbolic_reference - if self.rhodecode_vcs_repo.alias == 'svn': - adjusted_f_path = RepoFilesView.adjust_file_path_for_svn( - f_path, self.rhodecode_vcs_repo) + if self.rhodecode_vcs_repo.alias == "svn": + adjusted_f_path = RepoFilesView.adjust_file_path_for_svn(f_path, self.rhodecode_vcs_repo) if adjusted_f_path != f_path: log.debug( - 'Recognized svn tag or branch in file "%s", using svn ' - 'specific symbolic references', f_path) + 'Recognized svn tag or branch in file "%s", using svn ' "specific symbolic references", f_path + ) f_path = adjusted_f_path symbolic_reference = self._symbolic_reference_svn - branches = self._create_references( - self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, 'branch') + branches = self._create_references(self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, "branch") branches_group = (branches, _("Branches")) - tags = self._create_references( - self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, 'tag') + tags = self._create_references(self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, "tag") tags_group = (tags, _("Tags")) history.append(branches_group) @@ -1028,8 +1011,7 @@ class RepoFilesView(RepoAppView): return history, commits @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_file_history(self): self.load_default_context() @@ -1044,35 +1026,23 @@ class RepoFilesView(RepoAppView): for section_items, section in file_history: items = [] for obj_id, obj_text, obj_type in section_items: - at_rev = '' - if obj_type in ['branch', 'bookmark', 'tag']: + at_rev = "" + if obj_type in ["branch", "bookmark", "tag"]: at_rev = obj_text - entry = { - 'id': obj_id, - 'text': obj_text, - 'type': obj_type, - 'at_rev': at_rev - } + entry = {"id": obj_id, "text": obj_text, "type": obj_type, "at_rev": at_rev} items.append(entry) - res.append({ - 'text': section, - 'children': items - }) + res.append({"text": section, "children": items}) - data = { - 'more': False, - 'results': res - } + data = {"more": False, "results": res} return data - log.warning('Cannot fetch history for directory') + log.warning("Cannot fetch history for directory") raise HTTPBadRequest() @LoginRequired() - @HasRepoPermissionAnyDecorator( - 'repository.read', 'repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.read", "repository.write", "repository.admin") def repo_file_authors(self): c = self.load_default_context() @@ -1084,12 +1054,10 @@ class RepoFilesView(RepoAppView): raise HTTPBadRequest() c.file_last_commit = file_node.last_commit - if self.request.GET.get('annotate') == '1': + if self.request.GET.get("annotate") == "1": # use _hist from annotation if annotation mode is on commit_ids = {x[1] for x in file_node.annotate} - _hist = ( - self.rhodecode_vcs_repo.get_commit(commit_id) - for commit_id in commit_ids) + _hist = (self.rhodecode_vcs_repo.get_commit(commit_id) for commit_id in commit_ids) else: _f_history, _hist = self._get_node_history(commit, f_path) c.file_author = False @@ -1100,8 +1068,8 @@ class RepoFilesView(RepoAppView): if author not in unique: unique[commit.author] = [ h.email(author), - h.person(author, 'username_or_name_or_email'), - 1 # counter + h.person(author, "username_or_name_or_email"), + 1, # counter ] else: @@ -1113,20 +1081,21 @@ class RepoFilesView(RepoAppView): return self._get_template_context(c) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") def repo_files_check_head(self): self.load_default_context() commit_id, f_path, bytes_path = self._get_commit_and_path() - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) - new_path = self.request.POST.get('path') - operation = self.request.POST.get('operation') - path_exist = '' + new_path = self.request.POST.get("path") + operation = self.request.POST.get("operation") + path_exist = "" - if new_path and operation in ['create', 'upload']: - new_f_path = os.path.join(f_path.lstrip('/'), new_path) + if new_path and operation in ["create", "upload"]: + new_f_path = os.path.join(f_path.lstrip("/"), new_path) try: commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id) # NOTE(dan): construct whole path without leading / @@ -1136,24 +1105,19 @@ class RepoFilesView(RepoAppView): except (EmptyRepositoryError, NodeDoesNotExistError): pass - return { - 'branch': _branch_name, - 'sha': _sha_commit_id, - 'is_head': is_head, - 'path_exists': path_exist - } + return {"branch": _branch_name, "sha": _sha_commit_id, "is_head": is_head, "path_exists": path_exist} @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") def repo_files_remove_file(self): _ = self.request.translate c = self.load_default_context() commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path) self.check_branch_permission(_branch_name) @@ -1161,14 +1125,13 @@ class RepoFilesView(RepoAppView): c.commit = self._get_commit_or_redirect(commit_id) c.file = self._get_filenode_or_redirect(c.commit, bytes_path) - c.default_message = _( - 'Deleted file {} via RhodeCode Enterprise').format(f_path) + c.default_message = _("Deleted file {} via RhodeCode Enterprise").format(f_path) c.f_path = f_path return self._get_template_context(c) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") @CSRFRequired() def repo_files_delete_file(self): _ = self.request.translate @@ -1177,9 +1140,9 @@ class RepoFilesView(RepoAppView): commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path) self.check_branch_permission(_branch_name) @@ -1187,46 +1150,39 @@ class RepoFilesView(RepoAppView): c.commit = self._get_commit_or_redirect(commit_id) c.file = self._get_filenode_or_redirect(c.commit, bytes_path) - c.default_message = _('Deleted file {} via RhodeCode Enterprise').format(f_path) + c.default_message = _("Deleted file {} via RhodeCode Enterprise").format(f_path) c.f_path = f_path node_path = f_path author = self._rhodecode_db_user.full_contact - message = self.request.POST.get('message') or c.default_message + message = self.request.POST.get("message") or c.default_message try: - nodes = { - safe_bytes(node_path): { - 'content': b'' - } - } + nodes = {safe_bytes(node_path): {"content": b""}} ScmModel().delete_nodes( - user=self._rhodecode_db_user.user_id, repo=self.db_repo, + user=self._rhodecode_db_user.user_id, + repo=self.db_repo, message=message, nodes=nodes, parent_commit=c.commit, author=author, ) - h.flash( - _('Successfully deleted file `{}`').format( - h.escape(f_path)), category='success') + h.flash(_("Successfully deleted file `{}`").format(h.escape(f_path)), category="success") except Exception: - log.exception('Error during commit operation') - h.flash(_('Error occurred during commit'), category='error') - raise HTTPFound( - h.route_path('repo_commit', repo_name=self.db_repo_name, - commit_id='tip')) + log.exception("Error during commit operation") + h.flash(_("Error occurred during commit"), category="error") + raise HTTPFound(h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip")) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") def repo_files_edit_file(self): _ = self.request.translate c = self.load_default_context() commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path, commit_id=commit_id) self.check_branch_permission(_branch_name, commit_id=commit_id) @@ -1236,18 +1192,17 @@ class RepoFilesView(RepoAppView): if c.file.is_binary: files_url = h.route_path( - 'repo_files', - repo_name=self.db_repo_name, - commit_id=c.commit.raw_id, f_path=f_path) + "repo_files", repo_name=self.db_repo_name, commit_id=c.commit.raw_id, f_path=f_path + ) raise HTTPFound(files_url) - c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path) + c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path) c.f_path = f_path return self._get_template_context(c) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") @CSRFRequired() def repo_files_update_file(self): _ = self.request.translate @@ -1260,53 +1215,52 @@ class RepoFilesView(RepoAppView): c.file = self._get_filenode_or_redirect(c.commit, bytes_path) if c.file.is_binary: - raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, - commit_id=c.commit.raw_id, f_path=f_path)) + raise HTTPFound( + h.route_path("repo_files", repo_name=self.db_repo_name, commit_id=c.commit.raw_id, f_path=f_path) + ) - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path, commit_id=commit_id) self.check_branch_permission(_branch_name, commit_id=commit_id) - c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path) + c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path) c.f_path = f_path old_content = c.file.str_content sl = old_content.splitlines(1) - first_line = sl[0] if sl else '' + first_line = sl[0] if sl else "" r_post = self.request.POST # line endings: 0 - Unix, 1 - Mac, 2 - DOS line_ending_mode = detect_mode(first_line, 0) - content = convert_line_endings(r_post.get('content', ''), line_ending_mode) + content = convert_line_endings(r_post.get("content", ""), line_ending_mode) - message = r_post.get('message') or c.default_message + message = r_post.get("message") or c.default_message org_node_path = c.file.str_path - filename = r_post['filename'] + filename = r_post["filename"] root_path = c.file.dir_path pure_path = self.create_pure_path(root_path, filename) node_path = pure_path.as_posix() - default_redirect_url = h.route_path('repo_commit', repo_name=self.db_repo_name, - commit_id=commit_id) + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit_id) if content == old_content and node_path == org_node_path: - h.flash(_('No changes detected on {}').format(h.escape(org_node_path)), - category='warning') + h.flash(_("No changes detected on {}").format(h.escape(org_node_path)), category="warning") raise HTTPFound(default_redirect_url) try: mapping = { c.file.bytes_path: { - 'org_filename': org_node_path, - 'filename': safe_bytes(node_path), - 'content': safe_bytes(content), - 'lexer': '', - 'op': 'mod', - 'mode': c.file.mode + "org_filename": org_node_path, + "filename": safe_bytes(node_path), + "content": safe_bytes(content), + "lexer": "", + "op": "mod", + "mode": c.file.mode, } } @@ -1318,19 +1272,17 @@ class RepoFilesView(RepoAppView): parent_commit=c.commit, ) - h.flash(_('Successfully committed changes to file `{}`').format( - h.escape(f_path)), category='success') - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) + h.flash(_("Successfully committed changes to file `{}`").format(h.escape(f_path)), category="success") + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id) except Exception: - log.exception('Error occurred during commit') - h.flash(_('Error occurred during commit'), category='error') + log.exception("Error occurred during commit") + h.flash(_("Error occurred during commit"), category="error") raise HTTPFound(default_redirect_url) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") def repo_files_add_file(self): _ = self.request.translate c = self.load_default_context() @@ -1339,7 +1291,7 @@ class RepoFilesView(RepoAppView): self._ensure_not_locked() # Check if we need to use this page to upload binary - upload_binary = str2bool(self.request.params.get('upload_binary', False)) + upload_binary = str2bool(self.request.params.get("upload_binary", False)) c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False) if c.commit is None: @@ -1348,24 +1300,27 @@ class RepoFilesView(RepoAppView): if self.rhodecode_vcs_repo.is_empty(): # for empty repository we cannot check for current branch, we rely on # c.commit.branch instead - _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True + _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True else: - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path, commit_id=commit_id) self.check_branch_permission(_branch_name, commit_id=commit_id) - c.default_message = (_('Added file via RhodeCode Enterprise')) \ - if not upload_binary else (_('Edited file {} via RhodeCode Enterprise').format(f_path)) - c.f_path = f_path.lstrip('/') # ensure not relative path + c.default_message = ( + (_("Added file via RhodeCode Enterprise")) + if not upload_binary + else (_("Edited file {} via RhodeCode Enterprise").format(f_path)) + ) + c.f_path = f_path.lstrip("/") # ensure not relative path c.replace_binary = upload_binary return self._get_template_context(c) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") @CSRFRequired() def repo_files_create_file(self): _ = self.request.translate @@ -1380,56 +1335,48 @@ class RepoFilesView(RepoAppView): # calculate redirect URL if self.rhodecode_vcs_repo.is_empty(): - default_redirect_url = h.route_path( - 'repo_summary', repo_name=self.db_repo_name) + default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name) else: - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip") if self.rhodecode_vcs_repo.is_empty(): # for empty repository we cannot check for current branch, we rely on # c.commit.branch instead - _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True + _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True else: - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) self.forbid_non_head(is_head, f_path, commit_id=commit_id) self.check_branch_permission(_branch_name, commit_id=commit_id) - c.default_message = (_('Added file via RhodeCode Enterprise')) + c.default_message = _("Added file via RhodeCode Enterprise") c.f_path = f_path r_post = self.request.POST - message = r_post.get('message') or c.default_message - filename = r_post.get('filename') + message = r_post.get("message") or c.default_message + filename = r_post.get("filename") unix_mode = 0 if not filename: # If there's no commit, redirect to repo summary if type(c.commit) is EmptyCommit: - redirect_url = h.route_path( - 'repo_summary', repo_name=self.db_repo_name) + redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name) else: redirect_url = default_redirect_url - h.flash(_('No filename specified'), category='warning') + h.flash(_("No filename specified"), category="warning") raise HTTPFound(redirect_url) root_path = f_path pure_path = self.create_pure_path(root_path, filename) - node_path = pure_path.as_posix().lstrip('/') + node_path = pure_path.as_posix().lstrip("/") author = self._rhodecode_db_user.full_contact - content = convert_line_endings(r_post.get('content', ''), unix_mode) - nodes = { - safe_bytes(node_path): { - 'content': safe_bytes(content) - } - } + content = convert_line_endings(r_post.get("content", ""), unix_mode) + nodes = {safe_bytes(node_path): {"content": safe_bytes(content)}} try: - commit = ScmModel().create_nodes( user=self._rhodecode_db_user.user_id, repo=self.db_repo, @@ -1439,26 +1386,27 @@ class RepoFilesView(RepoAppView): author=author, ) - h.flash(_('Successfully committed new file `{}`').format(h.escape(node_path)), category='success') + h.flash(_("Successfully committed new file `{}`").format(h.escape(node_path)), category="success") - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id) except NonRelativePathError: - log.exception('Non Relative path found') - h.flash(_('The location specified must be a relative path and must not ' - 'contain .. in the path'), category='warning') + log.exception("Non Relative path found") + h.flash( + _("The location specified must be a relative path and must not " "contain .. in the path"), + category="warning", + ) raise HTTPFound(default_redirect_url) except (NodeError, NodeAlreadyExistsError) as e: - h.flash(h.escape(safe_str(e)), category='error') + h.flash(h.escape(safe_str(e)), category="error") except Exception: - log.exception('Error occurred during commit') - h.flash(_('Error occurred during commit'), category='error') + log.exception("Error occurred during commit") + h.flash(_("Error occurred during commit"), category="error") raise HTTPFound(default_redirect_url) @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") @CSRFRequired() def repo_files_upload_file(self): _ = self.request.translate @@ -1473,65 +1421,52 @@ class RepoFilesView(RepoAppView): # calculate redirect URL if self.rhodecode_vcs_repo.is_empty(): - default_redirect_url = h.route_path( - 'repo_summary', repo_name=self.db_repo_name) + default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name) else: - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip") if self.rhodecode_vcs_repo.is_empty(): # for empty repository we cannot check for current branch, we rely on # c.commit.branch instead - _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True + _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True else: - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) error = self.forbid_non_head(is_head, f_path, json_mode=True) if error: - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} error = self.check_branch_permission(_branch_name, json_mode=True) if error: - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} - c.default_message = (_('Added file via RhodeCode Enterprise')) + c.default_message = _("Added file via RhodeCode Enterprise") c.f_path = f_path r_post = self.request.POST message = c.default_message - user_message = r_post.getall('message') + user_message = r_post.getall("message") if isinstance(user_message, list) and user_message: # we take the first from duplicated results if it's not empty message = user_message[0] if user_message[0] else message nodes = {} - for file_obj in r_post.getall('files_upload') or []: + for file_obj in r_post.getall("files_upload") or []: content = file_obj.file filename = file_obj.filename root_path = f_path pure_path = self.create_pure_path(root_path, filename) - node_path = pure_path.as_posix().lstrip('/') + node_path = pure_path.as_posix().lstrip("/") - nodes[safe_bytes(node_path)] = { - 'content': content - } + nodes[safe_bytes(node_path)] = {"content": content} if not nodes: - error = 'missing files' - return { - 'error': error, - 'redirect_url': default_redirect_url - } + error = "missing files" + return {"error": error, "redirect_url": default_redirect_url} author = self._rhodecode_db_user.full_contact @@ -1545,49 +1480,35 @@ class RepoFilesView(RepoAppView): author=author, ) if len(nodes) == 1: - flash_message = _('Successfully committed {} new files').format(len(nodes)) + flash_message = _("Successfully committed {} new files").format(len(nodes)) else: - flash_message = _('Successfully committed 1 new file') + flash_message = _("Successfully committed 1 new file") - h.flash(flash_message, category='success') + h.flash(flash_message, category="success") - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id) except NonRelativePathError: - log.exception('Non Relative path found') - error = _('The location specified must be a relative path and must not ' - 'contain .. in the path') - h.flash(error, category='warning') + log.exception("Non Relative path found") + error = _("The location specified must be a relative path and must not " "contain .. in the path") + h.flash(error, category="warning") - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} except (NodeError, NodeAlreadyExistsError) as e: error = h.escape(e) - h.flash(error, category='error') + h.flash(error, category="error") - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} except Exception: - log.exception('Error occurred during commit') - error = _('Error occurred during commit') - h.flash(error, category='error') - return { - 'error': error, - 'redirect_url': default_redirect_url - } + log.exception("Error occurred during commit") + error = _("Error occurred during commit") + h.flash(error, category="error") + return {"error": error, "redirect_url": default_redirect_url} - return { - 'error': None, - 'redirect_url': default_redirect_url - } + return {"error": None, "redirect_url": default_redirect_url} @LoginRequired() - @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin') + @HasRepoPermissionAnyDecorator("repository.write", "repository.admin") @CSRFRequired() def repo_files_replace_file(self): _ = self.request.translate @@ -1601,64 +1522,50 @@ class RepoFilesView(RepoAppView): c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias) if self.rhodecode_vcs_repo.is_empty(): - default_redirect_url = h.route_path( - 'repo_summary', repo_name=self.db_repo_name) + default_redirect_url = h.route_path("repo_summary", repo_name=self.db_repo_name) else: - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id='tip') + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id="tip") if self.rhodecode_vcs_repo.is_empty(): # for empty repository we cannot check for current branch, we rely on # c.commit.branch instead - _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True + _branch_name, _sha_commit_id, is_head = c.commit.branch, "", True else: - _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + _branch_name, _sha_commit_id, is_head = self._is_valid_head( + commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name + ) error = self.forbid_non_head(is_head, f_path, json_mode=True) if error: - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} error = self.check_branch_permission(_branch_name, json_mode=True) if error: - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} - c.default_message = (_('Edited file {} via RhodeCode Enterprise').format(f_path)) + c.default_message = _("Edited file {} via RhodeCode Enterprise").format(f_path) c.f_path = f_path r_post = self.request.POST message = c.default_message - user_message = r_post.getall('message') + user_message = r_post.getall("message") if isinstance(user_message, list) and user_message: # we take the first from duplicated results if it's not empty message = user_message[0] if user_message[0] else message - data_for_replacement = r_post.getall('files_upload') or [] + data_for_replacement = r_post.getall("files_upload") or [] if (objects_count := len(data_for_replacement)) > 1: - return { - 'error': 'too many files for replacement', - 'redirect_url': default_redirect_url - } + return {"error": "too many files for replacement", "redirect_url": default_redirect_url} elif not objects_count: - return { - 'error': 'missing files', - 'redirect_url': default_redirect_url - } + return {"error": "missing files", "redirect_url": default_redirect_url} content = data_for_replacement[0].file retrieved_filename = data_for_replacement[0].filename - if retrieved_filename.split('.')[-1] != f_path.split('.')[-1]: + if retrieved_filename.split(".")[-1] != f_path.split(".")[-1]: return { - 'error': 'file extension of uploaded file doesn\'t match an original file\'s extension', - 'redirect_url': default_redirect_url + "error": "file extension of uploaded file doesn't match an original file's extension", + "redirect_url": default_redirect_url, } author = self._rhodecode_db_user.full_contact @@ -1669,36 +1576,26 @@ class RepoFilesView(RepoAppView): repo=self.db_repo, message=message, node={ - 'content': content, - 'file_path': f_path.encode(), + "content": content, + "file_path": f_path.encode(), }, parent_commit=c.commit, author=author, ) - h.flash(_('Successfully committed 1 new file'), category='success') + h.flash(_("Successfully committed 1 new file"), category="success") - default_redirect_url = h.route_path( - 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) + default_redirect_url = h.route_path("repo_commit", repo_name=self.db_repo_name, commit_id=commit.raw_id) except (NodeError, NodeAlreadyExistsError) as e: error = h.escape(e) - h.flash(error, category='error') + h.flash(error, category="error") - return { - 'error': error, - 'redirect_url': default_redirect_url - } + return {"error": error, "redirect_url": default_redirect_url} except Exception: - log.exception('Error occurred during commit') - error = _('Error occurred during commit') - h.flash(error, category='error') - return { - 'error': error, - 'redirect_url': default_redirect_url - } + log.exception("Error occurred during commit") + error = _("Error occurred during commit") + h.flash(error, category="error") + return {"error": error, "redirect_url": default_redirect_url} - return { - 'error': None, - 'redirect_url': default_redirect_url - } + return {"error": None, "redirect_url": default_redirect_url} diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py --- a/rhodecode/lib/rc_cache/__init__.py +++ b/rhodecode/lib/rc_cache/__init__.py @@ -55,7 +55,7 @@ register_backend( log = logging.getLogger(__name__) -FILE_TREE_CACHE_VER = 'v5' +FILE_TREE_CACHE_VER = 'v6' LICENSE_CACHE_VER = 'v3' PERMISSIONS_CACHE_VER = 'v2' diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -153,6 +153,7 @@ def remove_prefix(s, prefix): def find_calling_context(ignore_modules=None, depth=4, output_writer=None, indent=True): """ + How to find calling context: Look through the calling stack and return the frame which called this function and is part of core module ( ie. rhodecode.* ) diff --git a/rhodecode/lib/vcs/backends/git/commit.py b/rhodecode/lib/vcs/backends/git/commit.py --- a/rhodecode/lib/vcs/backends/git/commit.py +++ b/rhodecode/lib/vcs/backends/git/commit.py @@ -295,19 +295,16 @@ class GitCommit(base.BaseCommit): raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'") path = self._fix_path(path) - # call and check tree_id for this path - tree_id, _ = self._get_path_tree_id_and_type(path) - path_nodes = [] - for bytes_name, stat_, tree_item_id, node_kind in self._remote.tree_items(tree_id): + for obj_name, stat_, tree_item_id, node_kind, pre_load_data in self._remote.get_nodes(self.raw_id, path, pre_load): if node_kind is None: raise CommitError(f"Requested object type={node_kind} cannot be determined") - if path != b"": - obj_path = b"/".join((path, bytes_name)) + if path == b"": + obj_path = obj_name else: - obj_path = bytes_name + obj_path = b"/".join((path, obj_name)) # cache file mode for git, since we have it already if obj_path not in self._path_mode_cache: @@ -322,12 +319,12 @@ class GitCommit(base.BaseCommit): entry = self.nodes[obj_path] else: if node_kind == NodeKind.SUBMODULE: - url = self._get_submodule_url(b"/".join((path, bytes_name))) - entry= SubModuleNode(bytes_name, url=url, commit=tree_item_id, alias=self.repository.alias) + url = self._get_submodule_url(obj_path) + entry= SubModuleNode(obj_name, url=url, commit=tree_item_id, alias=self.repository.alias) elif node_kind == NodeKind.DIR: entry = DirNode(safe_bytes(obj_path), commit=self) elif node_kind == NodeKind.FILE: - entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) + entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load_data=pre_load_data) if entry: self.nodes[obj_path] = entry diff --git a/rhodecode/lib/vcs/backends/hg/commit.py b/rhodecode/lib/vcs/backends/hg/commit.py --- a/rhodecode/lib/vcs/backends/hg/commit.py +++ b/rhodecode/lib/vcs/backends/hg/commit.py @@ -274,7 +274,7 @@ class MercurialCommit(base.BaseCommit): path_nodes = [] - for obj_path, (node_kind, flags) in self._remote.dir_items(self.raw_id, path): + for obj_path, node_kind, flags, pre_load_data in self._remote.get_nodes(self.raw_id, path, pre_load): if node_kind is None: raise CommitError(f"Requested object type={node_kind} cannot be mapped to a proper type") @@ -295,7 +295,7 @@ class MercurialCommit(base.BaseCommit): if node_kind == NodeKind.DIR: entry = DirNode(safe_bytes(obj_path), commit=self) elif node_kind == NodeKind.FILE: - entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) + entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load, pre_load_data=pre_load_data) if entry: self.nodes[obj_path] = entry path_nodes.append(entry) diff --git a/rhodecode/lib/vcs/backends/svn/commit.py b/rhodecode/lib/vcs/backends/svn/commit.py --- a/rhodecode/lib/vcs/backends/svn/commit.py +++ b/rhodecode/lib/vcs/backends/svn/commit.py @@ -201,8 +201,8 @@ class SubversionCommit(base.BaseCommit): path = self._fix_path(path) path_nodes = [] - for name, node_kind in self._remote.get_nodes(self._svn_rev, path): - obj_path = vcspath.join(path, name) + + for obj_path, node_kind, pre_load_data in self._remote.get_nodes(self._svn_rev, path, pre_load): if node_kind is None: raise CommitError(f"Requested object type={node_kind} cannot be determined") @@ -224,7 +224,7 @@ class SubversionCommit(base.BaseCommit): if node_kind == NodeKind.DIR: entry = nodes.DirNode(safe_bytes(obj_path), commit=self) elif node_kind == NodeKind.FILE: - entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) + entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load, pre_load_data=pre_load_data) if entry: self.nodes[obj_path] = entry path_nodes.append(entry) diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py --- a/rhodecode/lib/vcs/nodes.py +++ b/rhodecode/lib/vcs/nodes.py @@ -241,7 +241,7 @@ class FileNode(Node): _filter_pre_load = [] - def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None): + def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None, pre_load_data=None): """ Only one of ``content`` and ``commit`` may be given. Passing both would raise ``NodeError`` exception. @@ -263,8 +263,10 @@ class FileNode(Node): content = safe_bytes(content) self._content = content self._mode = mode or FILEMODE_DEFAULT - - self._set_bulk_properties(pre_load) + if pre_load_data: + self._store_pre_load(pre_load_data) + else: + self._set_bulk_properties(pre_load) def __eq__(self, other): eq = super().__eq__(other) @@ -296,7 +298,10 @@ class FileNode(Node): remote = self.commit.get_remote() result = remote.bulk_file_request(self.commit.raw_id, self.bytes_path, pre_load) - for attr, value in result.items(): + self._store_pre_load(result.items()) + + def _store_pre_load(self, pre_load_data): + for attr, value in pre_load_data: if attr == "flags": self.__dict__["mode"] = safe_str(value) elif attr == "size": diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -1141,7 +1141,7 @@ class ReadmeFinder: self._default_renderer = default_renderer self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(default_renderer, []) - def search(self, commit, path=b'/'): + def search(self, commit, path=b'/', nodes=None): """ Find a readme in the given `commit`. """ @@ -1150,7 +1150,9 @@ class ReadmeFinder: if commit.get_node(bytes_path).kind != NodeKind.DIR: return None - nodes = commit.get_nodes(bytes_path) + if not nodes: + nodes = commit.get_nodes(bytes_path) + matches = self._match_readmes(nodes) matches = self._sort_according_to_priority(matches) if matches: diff --git a/rhodecode/templates/files/files_browser_tree.mako b/rhodecode/templates/files/files_browser_tree.mako --- a/rhodecode/templates/files/files_browser_tree.mako +++ b/rhodecode/templates/files/files_browser_tree.mako @@ -38,7 +38,7 @@ http://docker-dev:10020/ipython/files/ma <% has_files = False %> % if not c.file.is_submodule(): - % for cnt, node in enumerate(c.file): + % for cnt, node in enumerate(c.file_nodes): <% has_files = True %> @@ -105,6 +105,8 @@ http://docker-dev:10020/ipython/files/ma ${h.escape(c.file.name)} 
${c.file.url}
+ % else: +
${_('Empty directory')}
%endif diff --git a/rhodecode/tests/vcs/test_git.py b/rhodecode/tests/vcs/test_git.py --- a/rhodecode/tests/vcs/test_git.py +++ b/rhodecode/tests/vcs/test_git.py @@ -1218,19 +1218,21 @@ class TestGetSubmoduleUrl(object): def test_get_nodes_returns_links(self): repository = mock.MagicMock() repository.alias = "git" - repository._remote.tree_items.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE)] + # obj_name, stat_, tree_item_id, node_kind, pre_load_data + repository._remote.get_nodes.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE, [])] commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1) submodule_url = "https://code.rhodecode.com/dulwich" + get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.DIR)) get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url) with get_id_patch, get_submodule_patch as submodule_mock: nodes = commit.get_nodes(b"/abcde") - submodule_mock.assert_called_once_with(b"/abcde/subrepo") assert len(nodes) == 1 assert type(nodes[0]) == SubModuleNode assert nodes[0].url == submodule_url + submodule_mock.assert_called_once_with(b"/abcde/subrepo") class TestGetShadowInstance(object):