# HG changeset patch # User RhodeCode Admin # Date 2023-07-18 09:39:51 # Node ID 3f386a0484eb87683386a7bc2c20ea21ab1c1022 # Parent d5c2bc3477b61b9a6fa8fe0a1cebf646047bae1c apps: various fixes and improvements for python3 diff --git a/rhodecode/apps/_base/navigation.py b/rhodecode/apps/_base/navigation.py --- a/rhodecode/apps/_base/navigation.py +++ b/rhodecode/apps/_base/navigation.py @@ -1,4 +1,5 @@ +import dataclasses # Copyright (C) 2016-2020 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify @@ -30,8 +31,13 @@ from rhodecode.translation import _ log = logging.getLogger(__name__) -NavListEntry = collections.namedtuple( - 'NavListEntry', ['key', 'name', 'url', 'active_list']) + +@dataclasses.dataclass +class NavListEntry: + key: str + name: str + url: str + active_list: list class NavEntry(object): @@ -105,7 +111,7 @@ class NavigationRegistry(object): def __init__(self, labs_active=False): self._registered_entries = collections.OrderedDict() for item in self.__class__._base_entries: - self._registered_entries[item.key] = item + self.add_entry(item) if labs_active: self.add_entry(self._labs_entry) @@ -117,7 +123,8 @@ class NavigationRegistry(object): nav_list = [ NavListEntry(i.key, i.get_localized_name(request), i.generate_url(request), i.active_list) - for i in self._registered_entries.values()] + for i in self._registered_entries.values() + ] return nav_list diff --git a/rhodecode/apps/_base/subscribers.py b/rhodecode/apps/_base/subscribers.py --- a/rhodecode/apps/_base/subscribers.py +++ b/rhodecode/apps/_base/subscribers.py @@ -41,13 +41,12 @@ def trigger_user_permission_flush(event) automatic flush of permission caches, so the users affected receive new permissions Right Away """ - invalidate = True + affected_user_ids = set(event.user_ids) for user_id in affected_user_ids: for cache_namespace_uid_tmpl in cache_namespaces: cache_namespace_uid = cache_namespace_uid_tmpl.format(user_id) - del_keys = rc_cache.clear_cache_namespace( - 'cache_perms', cache_namespace_uid, invalidate=invalidate) + del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE) log.debug('Invalidated %s cache keys for user_id: %s and namespace %s', del_keys, user_id, cache_namespace_uid) diff --git a/rhodecode/apps/admin/views/repo_groups.py b/rhodecode/apps/admin/views/repo_groups.py --- a/rhodecode/apps/admin/views/repo_groups.py +++ b/rhodecode/apps/admin/views/repo_groups.py @@ -28,6 +28,7 @@ from pyramid.httpexceptions import HTTPF from pyramid.renderers import render from pyramid.response import Response +from sqlalchemy.orm import aliased from rhodecode import events from rhodecode.apps._base import BaseAppView, DataGridAppView @@ -160,6 +161,7 @@ class AdminRepoGroupsView(BaseAppView, D .count() repo_count = count(Repository.repo_id) + OwnerUser = aliased(User) base_q = Session.query( RepoGroup.group_name, RepoGroup.group_name_hash, @@ -167,19 +169,20 @@ class AdminRepoGroupsView(BaseAppView, D RepoGroup.group_id, RepoGroup.personal, RepoGroup.updated_on, - User, + OwnerUser.username.label('owner_username'), repo_count.label('repos_count') ) \ .filter(or_( # generate multiple IN to fix limitation problems *in_filter_generator(RepoGroup.group_id, allowed_ids) )) \ - .outerjoin(Repository, Repository.group_id == RepoGroup.group_id) \ - .join(User, User.user_id == RepoGroup.user_id) \ - .group_by(RepoGroup, User) + .outerjoin(Repository, RepoGroup.group_id == Repository.group_id) \ + .join(OwnerUser, RepoGroup.user_id == OwnerUser.user_id) + + base_q = base_q.group_by(RepoGroup, OwnerUser) if search_q: - like_expression = u'%{}%'.format(safe_unicode(search_q)) + like_expression = u'%{}%'.format(safe_str(search_q)) base_q = base_q.filter(or_( RepoGroup.group_name.ilike(like_expression), )) @@ -197,7 +200,7 @@ class AdminRepoGroupsView(BaseAppView, D sort_col = repo_count sort_defined = True elif order_by == 'user_username': - sort_col = User.username + sort_col = OwnerUser.username else: sort_col = getattr(RepoGroup, order_by, None) @@ -225,11 +228,10 @@ class AdminRepoGroupsView(BaseAppView, D "last_changeset_raw": "", "desc": desc(repo_gr.group_description, repo_gr.personal), - "owner": user_profile(repo_gr.User.username), + "owner": user_profile(repo_gr.owner_username), "top_level_repos": repo_gr.repos_count, "action": repo_group_actions( repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count), - } repo_groups_data.append(row) @@ -267,7 +269,7 @@ class AdminRepoGroupsView(BaseAppView, D if parent_group_id and _gr: if parent_group_id in [x[0] for x in c.repo_groups]: - parent_group_choice = safe_unicode(parent_group_id) + parent_group_choice = safe_str(parent_group_id) defaults.update({'group_parent_id': parent_group_choice}) @@ -297,7 +299,7 @@ class AdminRepoGroupsView(BaseAppView, D self._load_form_data(c) # permissions for can create group based on parent_id are checked # here in the Form - available_groups = map(lambda k: safe_unicode(k[0]), c.repo_groups) + available_groups = list(map(lambda k: safe_str(k[0]), c.repo_groups)) repo_group_form = RepoGroupForm( self.request.translate, available_groups=available_groups, can_create_in_root=can_create)() diff --git a/rhodecode/apps/admin/views/repositories.py b/rhodecode/apps/admin/views/repositories.py --- a/rhodecode/apps/admin/views/repositories.py +++ b/rhodecode/apps/admin/views/repositories.py @@ -26,6 +26,7 @@ from pyramid.httpexceptions import HTTPF from pyramid.renderers import render from pyramid.response import Response +from sqlalchemy.orm import aliased from rhodecode import events from rhodecode.apps._base import BaseAppView, DataGridAppView @@ -93,6 +94,8 @@ class AdminReposView(BaseAppView, DataGr ) \ .count() + RepoFork = aliased(Repository) + OwnerUser = aliased(User) base_q = Session.query( Repository.repo_id, Repository.repo_name, @@ -101,17 +104,17 @@ class AdminReposView(BaseAppView, DataGr Repository.repo_state, Repository.private, Repository.archived, - Repository.fork, Repository.updated_on, Repository._changeset_cache, - User, + RepoFork.repo_name.label('fork_repo_name'), + OwnerUser.username.label('owner_username'), ) \ .filter(or_( # generate multiple IN to fix limitation problems *in_filter_generator(Repository.repo_id, allowed_ids)) ) \ - .join(User, User.user_id == Repository.user_id) \ - .group_by(Repository, User) + .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \ + .join(OwnerUser, Repository.user_id == OwnerUser.user_id) if search_q: like_expression = u'%{}%'.format(safe_str(search_q)) @@ -119,6 +122,9 @@ class AdminReposView(BaseAppView, DataGr Repository.repo_name.ilike(like_expression), )) + #TODO: check if we need group_by here ? + #base_q = base_q.group_by(Repository, User) + repos_data_total_filtered_count = base_q.count() sort_defined = False @@ -126,7 +132,7 @@ class AdminReposView(BaseAppView, DataGr sort_col = func.lower(Repository.repo_name) sort_defined = True elif order_by == 'user_username': - sort_col = User.username + sort_col = OwnerUser.username else: sort_col = getattr(Repository, order_by, None) @@ -188,7 +194,7 @@ class AdminReposView(BaseAppView, DataGr if parent_group_id and _gr: if parent_group_id in [x[0] for x in c.repo_groups]: - parent_group_choice = safe_unicode(parent_group_id) + parent_group_choice = safe_str(parent_group_id) defaults.update({'repo_group': parent_group_choice}) diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -362,11 +362,11 @@ class UsersView(UserAppView): c = self.load_default_context() c.user = self.db_user - _repos = c.user.repositories - _repo_groups = c.user.repository_groups - _user_groups = c.user.user_groups - _pull_requests = c.user.user_pull_requests - _artifacts = c.user.artifacts + _repos = len(c.user.repositories) + _repo_groups = len(c.user.repository_groups) + _user_groups = len(c.user.user_groups) + _pull_requests = len(c.user.user_pull_requests) + _artifacts = len(c.user.artifacts) handle_repos = None handle_repo_groups = None @@ -378,46 +378,46 @@ class UsersView(UserAppView): def set_handle_flash_repos(): handle = handle_repos if handle == 'detach': - h.flash(_('Detached %s repositories') % len(_repos), + h.flash(_('Detached %s repositories') % _repos, category='success') elif handle == 'delete': - h.flash(_('Deleted %s repositories') % len(_repos), + h.flash(_('Deleted %s repositories') % _repos, category='success') def set_handle_flash_repo_groups(): handle = handle_repo_groups if handle == 'detach': - h.flash(_('Detached %s repository groups') % len(_repo_groups), + h.flash(_('Detached %s repository groups') % _repo_groups, category='success') elif handle == 'delete': - h.flash(_('Deleted %s repository groups') % len(_repo_groups), + h.flash(_('Deleted %s repository groups') % _repo_groups, category='success') def set_handle_flash_user_groups(): handle = handle_user_groups if handle == 'detach': - h.flash(_('Detached %s user groups') % len(_user_groups), + h.flash(_('Detached %s user groups') % _user_groups, category='success') elif handle == 'delete': - h.flash(_('Deleted %s user groups') % len(_user_groups), + h.flash(_('Deleted %s user groups') % _user_groups, category='success') def set_handle_flash_pull_requests(): handle = handle_pull_requests if handle == 'detach': - h.flash(_('Detached %s pull requests') % len(_pull_requests), + h.flash(_('Detached %s pull requests') % _pull_requests, category='success') elif handle == 'delete': - h.flash(_('Deleted %s pull requests') % len(_pull_requests), + h.flash(_('Deleted %s pull requests') % _pull_requests, category='success') def set_handle_flash_artifacts(): handle = handle_artifacts if handle == 'detach': - h.flash(_('Detached %s artifacts') % len(_artifacts), + h.flash(_('Detached %s artifacts') % _artifacts, category='success') elif handle == 'delete': - h.flash(_('Deleted %s artifacts') % len(_artifacts), + h.flash(_('Deleted %s artifacts') % _artifacts, category='success') handle_user = User.get_first_super_admin() diff --git a/rhodecode/apps/gist/views.py b/rhodecode/apps/gist/views.py --- a/rhodecode/apps/gist/views.py +++ b/rhodecode/apps/gist/views.py @@ -155,7 +155,6 @@ class GistView(BaseAppView): lifetime_options=[x[0] for x in c.lifetime_values]) try: - schema_data = schema.deserialize(data) # convert to safer format with just KEYs so we sure no duplicates @@ -263,8 +262,8 @@ class GistView(BaseAppView): raise HTTPNotFound() if return_format == 'raw': - content = '\n\n'.join([f.content for f in c.files - if (f_path is None or f.path == f_path)]) + content = b'\n\n'.join([f.content for f in c.files + if (f_path is None or f.path == f_path)]) response = Response(content) response.content_type = 'text/plain' return response diff --git a/rhodecode/apps/home/views.py b/rhodecode/apps/home/views.py --- a/rhodecode/apps/home/views.py +++ b/rhodecode/apps/home/views.py @@ -70,7 +70,7 @@ class HomeView(BaseAppView, DataGridAppV if skip_default_user and usr['username'] == UserModel.cls.DEFAULT_USER: return False return True - _users = filter(maybe_skip_default_user, _users) + _users = list(filter(maybe_skip_default_user, _users)) if include_groups: # extend with user groups @@ -229,7 +229,7 @@ class HomeView(BaseAppView, DataGridAppV 'value': org_query, 'value_display': 'user: `{}`'.format(obj.username), 'type': 'user', - 'icon_link': h.gravatar_url(obj.email, 30), + 'icon_link': h.gravatar_url(obj.email, 30, request=self.request), 'url': h.route_path( 'user_profile', username=obj.username) } diff --git a/rhodecode/apps/repo_group/views/repo_group_settings.py b/rhodecode/apps/repo_group/views/repo_group_settings.py --- a/rhodecode/apps/repo_group/views/repo_group_settings.py +++ b/rhodecode/apps/repo_group/views/repo_group_settings.py @@ -63,9 +63,8 @@ class RepoGroupSettingsView(RepoGroupApp show_empty_group=show_root_location) # filter out current repo group exclude_group_ids = [c.repo_group.group_id] - c.repo_groups = filter(lambda x: x[0] not in exclude_group_ids, - c.repo_groups) - c.repo_groups_choices = map(lambda k: k[0], c.repo_groups) + c.repo_groups = [x for x in c.repo_groups if x[0] not in exclude_group_ids] + c.repo_groups_choices = [k[0] for k in c.repo_groups] parent_group = c.repo_group.parent_group @@ -135,7 +134,7 @@ class RepoGroupSettingsView(RepoGroupApp schema = self._get_schema(c, old_values=old_values) c.form = RcForm(schema) - pstruct = self.request.POST.items() + pstruct = list(self.request.POST.items()) try: schema_data = c.form.validate(pstruct) diff --git a/rhodecode/apps/repository/__init__.py b/rhodecode/apps/repository/__init__.py --- a/rhodecode/apps/repository/__init__.py +++ b/rhodecode/apps/repository/__init__.py @@ -990,14 +990,14 @@ def includeme(config): route_name='edit_repo_fields_delete', request_method='POST', renderer='rhodecode:templates/admin/repos/repo_edit.mako') - # Locking + # quick actions: locking config.add_route( - name='repo_edit_toggle_locking', - pattern='/{repo_name:.*?[^/]}/settings/toggle_locking', repo_route=True) + name='repo_settings_quick_actions', + pattern='/{repo_name:.*?[^/]}/settings/quick-action', repo_route=True) config.add_view( RepoSettingsView, - attr='edit_advanced_toggle_locking', - route_name='repo_edit_toggle_locking', request_method='GET', + attr='repo_settings_quick_actions', + route_name='repo_settings_quick_actions', request_method='GET', renderer='rhodecode:templates/admin/repos/repo_edit.mako') # Remote diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -260,9 +260,10 @@ class RepoCommitsView(RepoAppView): ignore_whitespace=hide_whitespace_changes, context=diff_context) - diff_processor = diffs.DiffProcessor( - vcs_diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=c.fulldiff) + diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff', + diff_limit=diff_limit, + file_limit=file_limit, + show_full_diff=c.fulldiff) _parsed = diff_processor.prepare() @@ -285,9 +286,9 @@ class RepoCommitsView(RepoAppView): _diff = self.rhodecode_vcs_repo.get_diff( commit1, commit2, ignore_whitespace=hide_whitespace_changes, context=diff_context) - diff_processor = diffs.DiffProcessor( - _diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=c.fulldiff) + diff_processor = diffs.DiffProcessor(_diff, diff_format='newdiff', + diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=c.fulldiff) # downloads/raw we only need RAW diff nothing else diff = self.path_filter.get_raw_patch(diff_processor) c.changes[commit.raw_id] = [None, None, None, None, diff, None, None] @@ -643,17 +644,28 @@ class RepoCommitsView(RepoAppView): Session().commit() - return { + data = { 'store_fid': store_uid, 'access_path': h.route_path( 'download_file', fid=store_uid), 'fqn_access_path': h.route_url( 'download_file', fid=store_uid), - 'repo_access_path': h.route_path( - 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid), + # for EE those are replaced by FQN links on repo-only like + 'repo_access_path': h.route_url( + 'download_file', fid=store_uid), 'repo_fqn_access_path': h.route_url( - 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid), + 'download_file', fid=store_uid), } + # this data is a part of CE/EE additional code + if c.rhodecode_edition_id == 'EE': + data.update({ + 'repo_access_path': h.route_path( + 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid), + 'repo_fqn_access_path': h.route_url( + 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid), + }) + + return data @LoginRequired() @NotAnonymous() @@ -766,7 +778,7 @@ class RepoCommitsView(RepoAppView): 'comment_id': comment.comment_id, 'comment_version': comment_history.version, 'comment_author_username': comment_history.author.username, - 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16), + 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16, request=self.request), 'comment_created_on': h.age_component(comment_history.created_on, time_is_local=True), } diff --git a/rhodecode/apps/repository/views/repo_compare.py b/rhodecode/apps/repository/views/repo_compare.py --- a/rhodecode/apps/repository/views/repo_compare.py +++ b/rhodecode/apps/repository/views/repo_compare.py @@ -282,9 +282,10 @@ class RepoCompareView(RepoAppView): path=target_path, path1=source_path, ignore_whitespace=hide_whitespace_changes, context=diff_context) - diff_processor = diffs.DiffProcessor( - txt_diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=c.fulldiff) + diff_processor = diffs.DiffProcessor(txt_diff, diff_format='newdiff', + diff_limit=diff_limit, + file_limit=file_limit, + show_full_diff=c.fulldiff) _parsed = diff_processor.prepare() diffset = codeblocks.DiffSet( diff --git a/rhodecode/apps/repository/views/repo_feed.py b/rhodecode/apps/repository/views/repo_feed.py --- a/rhodecode/apps/repository/views/repo_feed.py +++ b/rhodecode/apps/repository/views/repo_feed.py @@ -17,8 +17,9 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import pytz + import logging +import datetime from pyramid.response import Response @@ -72,8 +73,9 @@ class RepoFeedView(RepoAppView): self.feed_items_per_page = config['feed_items_per_page'] def _changes(self, commit): - diff_processor = DiffProcessor( - commit.diff(), diff_limit=self.feed_diff_limit) + diff = commit.diff() + diff_processor = DiffProcessor(diff, diff_format='newdiff', + diff_limit=self.feed_diff_limit) _parsed = diff_processor.prepare(inline_diff=False) limited_diff = isinstance(_parsed, LimitedDiffContainer) @@ -97,7 +99,7 @@ class RepoFeedView(RepoAppView): has_hidden_changes=has_hidden_changes ) - def _set_timezone(self, date, tzinfo=pytz.utc): + def _set_timezone(self, date, tzinfo=datetime.timezone.utc): if not getattr(date, "tzinfo", None): date.replace(tzinfo=tzinfo) return date @@ -114,7 +116,10 @@ class RepoFeedView(RepoAppView): return list(collection[-self.feed_items_per_page:]) def uid(self, repo_id, commit_id): - return '{}:{}'.format(md5_safe(repo_id), md5_safe(commit_id)) + return '{}:{}'.format( + md5_safe(repo_id, return_type='str'), + md5_safe(commit_id, return_type='str') + ) @LoginRequired(auth_token_access=[UserApiKeys.ROLE_FEED]) @HasRepoPermissionAnyDecorator( @@ -126,7 +131,7 @@ class RepoFeedView(RepoAppView): self.load_default_context() force_recache = self.get_recache_flag() - cache_namespace_uid = 'cache_repo_feed.{}'.format(self.db_repo.repo_id) + cache_namespace_uid = 'repo_feed.{}'.format(self.db_repo.repo_id) condition = not (self.path_filter.is_enabled or force_recache) region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) @@ -144,7 +149,7 @@ class RepoFeedView(RepoAppView): for commit in reversed(self._get_commits()): date = self._set_timezone(commit.date) feed.add_item( - unique_id=self.uid(repo_id, commit.raw_id), + unique_id=self.uid(str(repo_id), commit.raw_id), title=self._get_title(commit), author_name=commit.author, description=self._get_description(commit), @@ -173,7 +178,7 @@ class RepoFeedView(RepoAppView): self.load_default_context() force_recache = self.get_recache_flag() - cache_namespace_uid = 'cache_repo_feed.{}'.format(self.db_repo.repo_id) + cache_namespace_uid = 'repo_feed.{}'.format(self.db_repo.repo_id) condition = not (self.path_filter.is_enabled or force_recache) region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) @@ -191,7 +196,7 @@ class RepoFeedView(RepoAppView): for commit in reversed(self._get_commits()): date = self._set_timezone(commit.date) feed.add_item( - unique_id=self.uid(repo_id, commit.raw_id), + unique_id=self.uid(str(repo_id), commit.raw_id), title=self._get_title(commit), author_name=commit.author, description=self._get_description(commit), diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -21,10 +21,10 @@ import itertools import logging import os -import shutil -import tempfile import collections -import urllib.request, urllib.parse, urllib.error +import urllib.request +import urllib.parse +import urllib.error import pathlib from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound @@ -38,12 +38,16 @@ from rhodecode.apps._base import RepoApp from rhodecode.lib import diffs, helpers as h, rc_cache from rhodecode.lib import audit_logger +from rhodecode.lib.hash_utils import sha1_safe +from rhodecode.lib.rc_cache.archive_cache import get_archival_cache_store, get_archival_config, ReentrantLock +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.view_utils import parse_path_ref from rhodecode.lib.exceptions import NonRelativePathError from rhodecode.lib.codeblocks import ( filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) -from rhodecode.lib.utils2 import ( - convert_line_endings, detect_mode, safe_str, str2bool, safe_int, sha1) +from rhodecode.lib.utils2 import convert_line_endings, detect_mode +from rhodecode.lib.type_utils import str2bool +from rhodecode.lib.str_utils import safe_str, safe_int from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) from rhodecode.lib.vcs import path as vcspath @@ -61,6 +65,48 @@ from rhodecode.model.db import Repositor log = logging.getLogger(__name__) +def get_archive_name(db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True): + # original backward compat name of archive + clean_name = safe_str(db_repo_name.replace('/', '_')) + + # e.g vcsserver-sub-1-abcfdef-archive-all.zip + # vcsserver-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip + + sub_repo = 'sub-1' if subrepos else 'sub-0' + commit = commit_sha if with_hash else 'archive' + path_marker = (path_sha if with_hash else '') or 'all' + archive_name = f'{clean_name}-{sub_repo}-{commit}-{path_marker}{ext}' + + return archive_name + + +def get_path_sha(at_path): + return safe_str(sha1_safe(at_path)[:8]) + + +def _get_archive_spec(fname): + log.debug('Detecting archive spec for: `%s`', fname) + + fileformat = None + ext = None + content_type = None + for a_type, content_type, extension in settings.ARCHIVE_SPECS: + + if fname.endswith(extension): + fileformat = a_type + log.debug('archive is of type: %s', fileformat) + ext = extension + break + + if not fileformat: + raise ValueError() + + # left over part of whole fname is the commit + commit_id = fname[:-len(ext)] + + return commit_id, ext, fileformat, content_type + + class RepoFilesView(RepoAppView): @staticmethod @@ -72,12 +118,12 @@ class RepoFilesView(RepoAppView): branches in the underlying repository. """ tags_and_branches = itertools.chain( - repo.branches.iterkeys(), - repo.tags.iterkeys()) + repo.branches.keys(), + repo.tags.keys()) tags_and_branches = sorted(tags_and_branches, key=len, reverse=True) for name in tags_and_branches: - if f_path.startswith('{}/'.format(name)): + if f_path.startswith(f'{name}/'): f_path = vcspath.relpath(f_path, name) break return f_path @@ -165,19 +211,23 @@ class RepoFilesView(RepoAppView): if not redirect_after: return None - _url = h.route_path( - 'repo_files_add_file', - repo_name=self.db_repo_name, commit_id=0, f_path='') - + add_new = upload_new = "" if h.HasRepoPermissionAny( 'repository.write', 'repository.admin')(self.db_repo_name): + _url = h.route_path( + 'repo_files_add_file', + repo_name=self.db_repo_name, commit_id=0, f_path='') add_new = h.link_to( - _('Click here to add a new file.'), _url, class_="alert-link") - else: - add_new = "" + _('add a new file'), _url, class_="alert-link") + + _url_upld = h.route_path( + 'repo_files_upload_file', + repo_name=self.db_repo_name, commit_id=0, f_path='') + upload_new = h.link_to( + _('upload a new file'), _url_upld, class_="alert-link") h.flash(h.literal( - _('There are no files yet. %s') % add_new), category='warning') + _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning') raise HTTPFound( h.route_path('repo_summary', repo_name=self.db_repo_name)) @@ -189,7 +239,7 @@ class RepoFilesView(RepoAppView): h.flash(h.escape(safe_str(e)), category='error') raise HTTPNotFound() - def _get_filenode_or_redirect(self, commit_obj, path): + def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None): """ Returns file_node, if error occurs or given path is directory, it'll redirect to top level path @@ -197,7 +247,7 @@ class RepoFilesView(RepoAppView): _ = self.request.translate try: - file_node = commit_obj.get_node(path) + file_node = commit_obj.get_node(path, pre_load=pre_load) if file_node.is_dir(): raise RepositoryError('The given path is a directory') except CommitDoesNotExistError: @@ -262,7 +312,7 @@ class RepoFilesView(RepoAppView): 'with caching: %s[TTL: %ss]' % ( repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) - cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + cache_namespace_uid = 'repo.{}'.format(repo_id) region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) @@ -279,28 +329,6 @@ class RepoFilesView(RepoAppView): rc_cache.FILE_TREE_CACHE_VER, self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev) - def _get_archive_spec(self, fname): - log.debug('Detecting archive spec for: `%s`', fname) - - fileformat = None - ext = None - content_type = None - for a_type, content_type, extension in settings.ARCHIVE_SPECS: - - if fname.endswith(extension): - fileformat = a_type - log.debug('archive is of type: %s', fileformat) - ext = extension - break - - if not fileformat: - raise ValueError() - - # left over part of whole fname is the commit - commit_id = fname[:-len(ext)] - - return commit_id, ext, fileformat, content_type - def create_pure_path(self, *parts): # Split paths and sanitize them, removing any ../ etc sanitized_path = [ @@ -325,22 +353,6 @@ class RepoFilesView(RepoAppView): return lf_enabled - def _get_archive_name(self, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True): - # original backward compat name of archive - clean_name = safe_str(db_repo_name.replace('/', '_')) - - # e.g vcsserver.zip - # e.g vcsserver-abcdefgh.zip - # e.g vcsserver-abcdefgh-defghijk.zip - archive_name = '{}{}{}{}{}{}'.format( - clean_name, - '-sub' if subrepos else '', - commit_sha, - '-{}'.format('plain') if not with_hash else '', - '-{}'.format(path_sha) if path_sha else '', - ext) - return archive_name - @LoginRequired() @HasRepoPermissionAnyDecorator( 'repository.read', 'repository.write', 'repository.admin') @@ -360,7 +372,7 @@ class RepoFilesView(RepoAppView): try: commit_id, ext, fileformat, content_type = \ - self._get_archive_spec(fname) + _get_archive_spec(fname) except ValueError: return Response(_('Unknown archive type for: `{}`').format( h.escape(fname))) @@ -383,94 +395,86 @@ class RepoFilesView(RepoAppView): except Exception: return Response(_('No node at path {} for this repository').format(h.escape(at_path))) - # path sha is part of subdir - path_sha = '' - if at_path != default_at_path: - path_sha = sha1(at_path)[:8] - short_sha = '-{}'.format(safe_str(commit.short_id)) - # used for cache etc - archive_name = self._get_archive_name( - self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos, - path_sha=path_sha, with_hash=with_hash) + path_sha = get_path_sha(at_path) + + # used for cache etc, consistent unique archive name + archive_name_key = get_archive_name( + self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, + path_sha=path_sha, with_hash=True) if not with_hash: - short_sha = '' path_sha = '' # what end client gets served - response_archive_name = self._get_archive_name( - self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos, + response_archive_name = get_archive_name( + self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos, path_sha=path_sha, with_hash=with_hash) + # remove extension from our archive directory name archive_dir_name = response_archive_name[:-len(ext)] - use_cached_archive = False - archive_cache_dir = CONFIG.get('archive_cache_dir') - archive_cache_enabled = archive_cache_dir and not self.request.GET.get('no_cache') - cached_archive_path = None + archive_cache_disable = self.request.GET.get('no_cache') + + d_cache = get_archival_cache_store(config=CONFIG) + # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver + d_cache_conf = get_archival_config(config=CONFIG) - if archive_cache_enabled: - # check if we it's ok to write, and re-create the archive cache - if not os.path.isdir(CONFIG['archive_cache_dir']): - os.makedirs(CONFIG['archive_cache_dir']) - - cached_archive_path = os.path.join( - CONFIG['archive_cache_dir'], archive_name) - if os.path.isfile(cached_archive_path): - log.debug('Found cached archive in %s', cached_archive_path) - fd, archive = None, cached_archive_path + reentrant_lock_key = archive_name_key + '.lock' + with ReentrantLock(d_cache, reentrant_lock_key): + # This is also a cache key + use_cached_archive = False + if archive_name_key in d_cache and not archive_cache_disable: + reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True) use_cached_archive = True + log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s', + archive_name_key, tag, reader.name) else: - log.debug('Archive %s is not yet cached', archive_name) + reader = None + log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key) - # generate new archive, as previous was not found in the cache - if not use_cached_archive: - _dir = os.path.abspath(archive_cache_dir) if archive_cache_dir else None - fd, archive = tempfile.mkstemp(dir=_dir) - log.debug('Creating new temp archive in %s', archive) - try: - commit.archive_repo(archive, archive_dir_name=archive_dir_name, - kind=fileformat, subrepos=subrepos, - archive_at_path=at_path) - except ImproperArchiveTypeError: - return _('Unknown archive type') - if archive_cache_enabled: - # if we generated the archive and we have cache enabled - # let's use this for future - log.debug('Storing new archive in %s', cached_archive_path) - shutil.move(archive, cached_archive_path) - archive = cached_archive_path + # generate new archive, as previous was not found in the cache + if not reader: + # first remove expired items, before generating a new one :) + # we di this manually because automatic eviction is disabled + d_cache.cull(retry=True) + + try: + commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name, + kind=fileformat, subrepos=subrepos, + archive_at_path=at_path, cache_config=d_cache_conf) + except ImproperArchiveTypeError: + return _('Unknown archive type') - # store download action - audit_logger.store_web( - 'repo.archive.download', action_data={ - 'user_agent': self.request.user_agent, - 'archive_name': archive_name, - 'archive_spec': fname, - 'archive_cached': use_cached_archive}, - user=self._rhodecode_user, - repo=self.db_repo, - commit=True - ) + reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True) + + if not reader: + raise ValueError('archive cache reader is empty, failed to fetch file from distributed archive cache') - def get_chunked_archive(archive_path): - with open(archive_path, 'rb') as stream: - while True: - data = stream.read(16 * 1024) - if not data: - if fd: # fd means we used temporary file - os.close(fd) - if not archive_cache_enabled: - log.debug('Destroying temp archive %s', archive_path) - os.remove(archive_path) - break - yield data + def archive_iterator(_reader): + while 1: + data = _reader.read(1024) + if not data: + break + yield data - response = Response(app_iter=get_chunked_archive(archive)) - response.content_disposition = str('attachment; filename=%s' % response_archive_name) + response = Response(app_iter=archive_iterator(reader)) + response.content_disposition = f'attachment; filename={response_archive_name}' response.content_type = str(content_type) - return response + try: + return response + finally: + # store download action + audit_logger.store_web( + 'repo.archive.download', action_data={ + 'user_agent': self.request.user_agent, + 'archive_name': archive_name_key, + 'archive_spec': fname, + 'archive_cached': use_cached_archive}, + user=self._rhodecode_user, + repo=self.db_repo, + commit=True + ) def _get_file_node(self, commit_id, f_path): if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]: @@ -478,8 +482,7 @@ class RepoFilesView(RepoAppView): try: node = commit.get_node(f_path) if node.is_dir(): - raise NodeError('%s path is a %s not a file' - % (node, type(node))) + raise NodeError(f'{node} path is a {type(node)} not a file') except NodeDoesNotExistError: commit = EmptyCommit( commit_id=commit_id, @@ -489,12 +492,12 @@ class RepoFilesView(RepoAppView): message=commit.message, author=commit.author, date=commit.date) - node = FileNode(f_path, '', commit=commit) + node = FileNode(safe_bytes(f_path), b'', commit=commit) else: commit = EmptyCommit( repo=self.rhodecode_vcs_repo, alias=self.rhodecode_vcs_repo.alias) - node = FileNode(f_path, '', commit=commit) + node = FileNode(safe_bytes(f_path), b'', commit=commit) return node @LoginRequired() @@ -551,12 +554,13 @@ class RepoFilesView(RepoAppView): _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context) - diff = diffs.DiffProcessor(_diff, format='gitdiff') + # NOTE: this was using diff_format='gitdiff' + diff = diffs.DiffProcessor(_diff, diff_format='newdiff') response = Response(self.path_filter.get_raw_patch(diff)) response.content_type = 'text/plain' response.content_disposition = ( - 'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2) + f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff' ) charset = self._get_default_encoding(c) if charset: @@ -567,7 +571,8 @@ class RepoFilesView(RepoAppView): _diff = diffs.get_gitdiff(node1, node2, ignore_whitespace=ignore_whitespace, context=line_context) - diff = diffs.DiffProcessor(_diff, format='gitdiff') + # NOTE: this was using diff_format='gitdiff' + diff = diffs.DiffProcessor(_diff, diff_format='newdiff') response = Response(self.path_filter.get_raw_patch(diff)) response.content_type = 'text/plain' @@ -637,8 +642,7 @@ class RepoFilesView(RepoAppView): c.annotate = view_name == 'repo_files:annotated' # default is false, but .rst/.md files later are auto rendered, we can # overwrite auto rendering by setting this GET flag - c.renderer = view_name == 'repo_files:rendered' or \ - not self.request.GET.get('no-render', False) + c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False) commit_id, f_path = self._get_commit_and_path() @@ -675,7 +679,7 @@ class RepoFilesView(RepoAppView): # files or dirs try: - c.file = c.commit.get_node(f_path) + c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data']) c.file_author = True c.file_tree = '' @@ -917,7 +921,7 @@ class RepoFilesView(RepoAppView): 'with caching: %s[TTL: %ss]' % ( repo_id, commit_id, f_path, cache_on, cache_seconds or 0)) - cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + cache_namespace_uid = 'repo.{}'.format(repo_id) region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid) @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) @@ -950,7 +954,7 @@ class RepoFilesView(RepoAppView): metadata = self._get_nodelist_at_commit( self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path) - return {'nodes': metadata} + return {'nodes': [x for x in metadata]} def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type): items = [] @@ -967,7 +971,7 @@ class RepoFilesView(RepoAppView): # NOTE(dan): old code we used in "diff" mode compare new_f_path = vcspath.join(name, f_path) - return u'%s@%s' % (new_f_path, commit_id) + return f'{new_f_path}@{commit_id}' def _get_node_history(self, commit_obj, f_path, commits=None): """ @@ -1194,8 +1198,8 @@ class RepoFilesView(RepoAppView): message = self.request.POST.get('message') or c.default_message try: nodes = { - node_path: { - 'content': '' + safe_bytes(node_path): { + 'content': b'' } } ScmModel().delete_nodes( @@ -1273,7 +1277,7 @@ class RepoFilesView(RepoAppView): c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path) c.f_path = f_path - old_content = c.file.content + old_content = c.file.str_content sl = old_content.splitlines(1) first_line = sl[0] if sl else '' @@ -1283,7 +1287,8 @@ class RepoFilesView(RepoAppView): content = convert_line_endings(r_post.get('content', ''), line_ending_mode) message = r_post.get('message') or c.default_message - org_node_path = c.file.unicode_path + + org_node_path = c.file.str_path filename = r_post['filename'] root_path = c.file.dir_path @@ -1299,10 +1304,10 @@ class RepoFilesView(RepoAppView): try: mapping = { - org_node_path: { + c.file.bytes_path: { 'org_filename': org_node_path, - 'filename': node_path, - 'content': content, + 'filename': safe_bytes(node_path), + 'content': safe_bytes(content), 'lexer': '', 'op': 'mod', 'mode': c.file.mode @@ -1400,8 +1405,6 @@ class RepoFilesView(RepoAppView): filename = r_post.get('filename') unix_mode = 0 - content = convert_line_endings(r_post.get('content', ''), unix_mode) - if not filename: # If there's no commit, redirect to repo summary if type(c.commit) is EmptyCommit: @@ -1417,9 +1420,10 @@ class RepoFilesView(RepoAppView): node_path = pure_path.as_posix().lstrip('/') author = self._rhodecode_db_user.full_contact + content = convert_line_endings(r_post.get('content', ''), unix_mode) nodes = { - node_path: { - 'content': content + safe_bytes(node_path): { + 'content': safe_bytes(content) } } @@ -1518,7 +1522,7 @@ class RepoFilesView(RepoAppView): pure_path = self.create_pure_path(root_path, filename) node_path = pure_path.as_posix().lstrip('/') - nodes[node_path] = { + nodes[safe_bytes(node_path)] = { 'content': content } diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -234,8 +234,7 @@ class RepoPullRequestsView(RepoAppView, source_repo, source_ref_id, target_ref_id, hide_whitespace_changes, diff_context) - diff_processor = diffs.DiffProcessor( - vcs_diff, format='newdiff', diff_limit=diff_limit, + diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff', diff_limit=diff_limit, file_limit=file_limit, show_full_diff=fulldiff) _parsed = diff_processor.prepare() @@ -259,9 +258,9 @@ class RepoPullRequestsView(RepoAppView, ignore_whitespace=hide_whitespace_changes, context=diff_context) - diff_processor = diffs.DiffProcessor( - vcs_diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=fulldiff) + diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff', + diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=fulldiff) _parsed = diff_processor.prepare() @@ -933,7 +932,7 @@ class RepoPullRequestsView(RepoAppView, .filter(Repository.fork_id == self.db_repo.parent.repo_id) if filter_query: - ilike_expression = u'%{}%'.format(safe_unicode(filter_query)) + ilike_expression = u'%{}%'.format(safe_str(filter_query)) parents_query = parents_query.filter( Repository.repo_name.ilike(ilike_expression)) parents = parents_query.limit(20).all() diff --git a/rhodecode/apps/repository/views/repo_settings.py b/rhodecode/apps/repository/views/repo_settings.py --- a/rhodecode/apps/repository/views/repo_settings.py +++ b/rhodecode/apps/repository/views/repo_settings.py @@ -249,3 +249,29 @@ class RepoSettingsView(RepoAppView): category='error') raise HTTPFound( h.route_path('edit_repo_statistics', repo_name=self.db_repo_name)) + + @LoginRequired() + @HasRepoPermissionAnyDecorator('repository.admin') + def repo_settings_quick_actions(self): + _ = self.request.translate + + set_lock = self.request.GET.get('set_lock') + set_unlock = self.request.GET.get('set_unlock') + + try: + if set_lock: + Repository.lock(self.db_repo, self._rhodecode_user.user_id, + lock_reason=Repository.LOCK_WEB) + h.flash(_('Locked repository'), category='success') + elif set_unlock: + Repository.unlock(self.db_repo) + h.flash(_('Unlocked repository'), category='success') + except Exception as e: + log.exception("Exception during unlocking") + h.flash(_('An error occurred during unlocking'), category='error') + + raise HTTPFound( + h.route_path('repo_summary', repo_name=self.db_repo_name)) + + + diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py --- a/rhodecode/apps/ssh_support/lib/backends/__init__.py +++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py @@ -63,23 +63,23 @@ class SshWrapper(object): from rhodecode.model.meta import raw_query_executor, Base table = Table('user_ssh_keys', Base.metadata, autoload=False) + atime = datetime.datetime.utcnow() stmt = ( table.update() .where(table.c.ssh_key_id == key_id) - .values(accessed_on=datetime.datetime.utcnow()) - .returning(table.c.accessed_on, table.c.ssh_key_fingerprint) + .values(accessed_on=atime) + # no MySQL Support for .returning :(( + #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint) ) - scalar_res = None + res_count = None with raw_query_executor() as session: result = session.execute(stmt) if result.rowcount: - scalar_res = result.first() + res_count = result.rowcount - if scalar_res: - atime, ssh_key_fingerprint = scalar_res - log.debug('Update key id:`%s` fingerprint:`%s` access time', - key_id, ssh_key_fingerprint) + if res_count: + log.debug('Update key id:`%s` access time', key_id) def get_user(self, user_id): user = AttributeDict() diff --git a/rhodecode/apps/ssh_support/utils.py b/rhodecode/apps/ssh_support/utils.py --- a/rhodecode/apps/ssh_support/utils.py +++ b/rhodecode/apps/ssh_support/utils.py @@ -40,8 +40,8 @@ SSH_OPTS = 'no-pty,no-port-forwarding,no def get_all_active_keys(): result = UserSshKeys.query() \ - .options(joinedload(UserSshKeys.user)) \ - .filter(UserSshKeys.user != User.get_default_user()) \ + .join(User) \ + .filter(User != User.get_default_user()) \ .filter(User.active == true()) \ .all() return result @@ -55,6 +55,10 @@ def _generate_ssh_authorized_keys_file( os.path.expanduser(authorized_keys_file_path)) tmp_file_dir = os.path.dirname(authorized_keys_file_path) + if not os.path.exists(tmp_file_dir): + log.debug('SSH authorized_keys file dir does not exist, creating one now...') + os.makedirs(tmp_file_dir) + all_active_keys = get_all_active_keys() if allow_shell: @@ -65,6 +69,7 @@ def _generate_ssh_authorized_keys_file( if not os.path.isfile(authorized_keys_file_path): log.debug('Creating file at %s', authorized_keys_file_path) with open(authorized_keys_file_path, 'w'): + # create a file with write access pass if not os.access(authorized_keys_file_path, os.R_OK): @@ -78,7 +83,7 @@ def _generate_ssh_authorized_keys_file( dir=tmp_file_dir) now = datetime.datetime.utcnow().isoformat() - keys_file = os.fdopen(fd, 'wb') + keys_file = os.fdopen(fd, 'wt') keys_file.write(HEADER.format(len(all_active_keys), now)) ini_path = rhodecode.CONFIG['__file__'] diff --git a/rhodecode/apps/user_group/views/__init__.py b/rhodecode/apps/user_group/views/__init__.py --- a/rhodecode/apps/user_group/views/__init__.py +++ b/rhodecode/apps/user_group/views/__init__.py @@ -75,7 +75,7 @@ class UserGroupsView(UserGroupAppView): 'first_name': user.first_name, 'last_name': user.last_name, 'username': user.username, - 'icon_link': h.gravatar_url(user.email, 30), + 'icon_link': h.gravatar_url(user.email, 30, request=self.request), 'value_display': h.person(user.email), 'value': user.username, 'value_type': 'user',