apps: various fixes and improvements for python3

rhodecode/apps/_base/navigation.py

@@ -1,4 +1,5 b''
 
+import dataclasses
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -30,8 +31,13 b' from rhodecode.translation import _'
 
 log = logging.getLogger(__name__)
 
-NavListEntry = collections.namedtuple(
-    'NavListEntry', ['key', 'name', 'url', 'active_list'])
+
[e-mail]
+class NavListEntry:
+    key: str
+    name: str
+    url: str
+    active_list: list
 
 
 class NavEntry(object):
@@ -105,7 +111,7 b' class NavigationRegistry(object):'
     def __init__(self, labs_active=False):
         self._registered_entries = collections.OrderedDict()
         for item in self.__class__._base_entries:
-            self._registered_entries[item.key] = item
+            self.add_entry(item)
 
         if labs_active:
             self.add_entry(self._labs_entry)
@@ -117,7 +123,8 b' class NavigationRegistry(object):'
         nav_list = [
             NavListEntry(i.key, i.get_localized_name(request),
                          i.generate_url(request), i.active_list)
-            for i in self._registered_entries.values()]
+            for i in self._registered_entries.values()
+        ]
         return nav_list
 
 

rhodecode/apps/_base/subscribers.py

@@ -41,13 +41,12 b' def trigger_user_permission_flush(event)'
     automatic flush of permission caches, so the users affected receive new permissions
     Right Away
     """
-    invalidate = True
+
     affected_user_ids = set(event.user_ids)
     for user_id in affected_user_ids:
         for cache_namespace_uid_tmpl in cache_namespaces:
             cache_namespace_uid = cache_namespace_uid_tmpl.format(user_id)
-            del_keys = rc_cache.clear_cache_namespace(
-                'cache_perms', cache_namespace_uid, invalidate=invalidate)
+            del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE)
             log.debug('Invalidated %s cache keys for user_id: %s and namespace %s',
                       del_keys, user_id, cache_namespace_uid)
 

rhodecode/apps/admin/views/repo_groups.py

@@ -28,6 +28,7 b' from pyramid.httpexceptions import HTTPF'
 
 from pyramid.renderers import render
 from pyramid.response import Response
+from sqlalchemy.orm import aliased
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
@@ -160,6 +161,7 b' class AdminRepoGroupsView(BaseAppView, D'
             .count()
 
         repo_count = count(Repository.repo_id)
+        OwnerUser = aliased(User)
         base_q = Session.query(
             RepoGroup.group_name,
             RepoGroup.group_name_hash,
@@ -167,19 +169,20 b' class AdminRepoGroupsView(BaseAppView, D'
             RepoGroup.group_id,
             RepoGroup.personal,
             RepoGroup.updated_on,
-            User,
+            OwnerUser.username.label('owner_username'),
             repo_count.label('repos_count')
             ) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             )) \
-            .outerjoin(Repository,  Repository.group_id == RepoGroup.group_id) \
-            .join(User, User.user_id == RepoGroup.user_id) \
-            .group_by(RepoGroup, User)
+            .outerjoin(Repository,  RepoGroup.group_id == Repository.group_id) \
+            .join(OwnerUser,  RepoGroup.user_id == OwnerUser.user_id)
+
+        base_q = base_q.group_by(RepoGroup, OwnerUser)
 
         if search_q:
-            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            like_expression = u'%{}%'.format(safe_str(search_q))
             base_q = base_q.filter(or_(
                 RepoGroup.group_name.ilike(like_expression),
             ))
@@ -197,7 +200,7 b' class AdminRepoGroupsView(BaseAppView, D'
             sort_col = repo_count
             sort_defined = True
         elif order_by == 'user_username':
-            sort_col = User.username
+            sort_col = OwnerUser.username
         else:
             sort_col = getattr(RepoGroup, order_by, None)
 
@@ -225,11 +228,10 b' class AdminRepoGroupsView(BaseAppView, D'
                 "last_changeset_raw": "",
 
                 "desc": desc(repo_gr.group_description, repo_gr.personal),
-                "owner": user_profile(repo_gr.User.username),
+                "owner": user_profile(repo_gr.owner_username),
                 "top_level_repos": repo_gr.repos_count,
                 "action": repo_group_actions(
                     repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count),
-
             }
 
             repo_groups_data.append(row)
@@ -267,7 +269,7 b' class AdminRepoGroupsView(BaseAppView, D'
 
         if parent_group_id and _gr:
             if parent_group_id in [x[0] for x in c.repo_groups]:
-                parent_group_choice = safe_unicode(parent_group_id)
+                parent_group_choice = safe_str(parent_group_id)
 
         defaults.update({'group_parent_id': parent_group_choice})
 
@@ -297,7 +299,7 b' class AdminRepoGroupsView(BaseAppView, D'
         self._load_form_data(c)
         # permissions for can create group based on parent_id are checked
         # here in the Form
-        available_groups = map(lambda k: safe_unicode(k[0]), c.repo_groups)
+        available_groups = list(map(lambda k: safe_str(k[0]), c.repo_groups))
         repo_group_form = RepoGroupForm(
             self.request.translate, available_groups=available_groups,
             can_create_in_root=can_create)()

rhodecode/apps/admin/views/repositories.py

@@ -26,6 +26,7 b' from pyramid.httpexceptions import HTTPF'
 
 from pyramid.renderers import render
 from pyramid.response import Response
+from sqlalchemy.orm import aliased
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
@@ -93,6 +94,8 b' class AdminReposView(BaseAppView, DataGr'
             ) \
             .count()
 
+        RepoFork = aliased(Repository)
+        OwnerUser = aliased(User)
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
@@ -101,17 +104,17 b' class AdminReposView(BaseAppView, DataGr'
             Repository.repo_state,
             Repository.private,
             Repository.archived,
-            Repository.fork,
             Repository.updated_on,
             Repository._changeset_cache,
-            User,
+            RepoFork.repo_name.label('fork_repo_name'),
+            OwnerUser.username.label('owner_username'),
             ) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
-            .join(User, User.user_id == Repository.user_id) \
-            .group_by(Repository, User)
+            .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \
+            .join(OwnerUser, Repository.user_id == OwnerUser.user_id)
 
         if search_q:
             like_expression = u'%{}%'.format(safe_str(search_q))
@@ -119,6 +122,9 b' class AdminReposView(BaseAppView, DataGr'
                 Repository.repo_name.ilike(like_expression),
             ))
 
+        #TODO: check if we need group_by here ?
+        #base_q = base_q.group_by(Repository, User)
+
         repos_data_total_filtered_count = base_q.count()
 
         sort_defined = False
@@ -126,7 +132,7 b' class AdminReposView(BaseAppView, DataGr'
             sort_col = func.lower(Repository.repo_name)
             sort_defined = True
         elif order_by == 'user_username':
-            sort_col = User.username
+            sort_col = OwnerUser.username
         else:
             sort_col = getattr(Repository, order_by, None)
 
@@ -188,7 +194,7 b' class AdminReposView(BaseAppView, DataGr'
 
         if parent_group_id and _gr:
             if parent_group_id in [x[0] for x in c.repo_groups]:
-                parent_group_choice = safe_unicode(parent_group_id)
+                parent_group_choice = safe_str(parent_group_id)
 
         defaults.update({'repo_group': parent_group_choice})
 

rhodecode/apps/admin/views/users.py

@@ -362,11 +362,11 b' class UsersView(UserAppView):'
         c = self.load_default_context()
         c.user = self.db_user
 
-        _repos = c.user.repositories
-        _repo_groups = c.user.repository_groups
-        _user_groups = c.user.user_groups
-        _pull_requests = c.user.user_pull_requests
-        _artifacts = c.user.artifacts
+        _repos = len(c.user.repositories)
+        _repo_groups = len(c.user.repository_groups)
+        _user_groups = len(c.user.user_groups)
+        _pull_requests = len(c.user.user_pull_requests)
+        _artifacts = len(c.user.artifacts)
 
         handle_repos = None
         handle_repo_groups = None
@@ -378,46 +378,46 b' class UsersView(UserAppView):'
         def set_handle_flash_repos():
             handle = handle_repos
             if handle == 'detach':
-                h.flash(_('Detached %s repositories') % len(_repos),
+                h.flash(_('Detached %s repositories') % _repos,
                         category='success')
             elif handle == 'delete':
-                h.flash(_('Deleted %s repositories') % len(_repos),
+                h.flash(_('Deleted %s repositories') % _repos,
                         category='success')
 
         def set_handle_flash_repo_groups():
             handle = handle_repo_groups
             if handle == 'detach':
-                h.flash(_('Detached %s repository groups') % len(_repo_groups),
+                h.flash(_('Detached %s repository groups') % _repo_groups,
                         category='success')
             elif handle == 'delete':
-                h.flash(_('Deleted %s repository groups') % len(_repo_groups),
+                h.flash(_('Deleted %s repository groups') % _repo_groups,
                         category='success')
 
         def set_handle_flash_user_groups():
             handle = handle_user_groups
             if handle == 'detach':
-                h.flash(_('Detached %s user groups') % len(_user_groups),
+                h.flash(_('Detached %s user groups') % _user_groups,
                         category='success')
             elif handle == 'delete':
-                h.flash(_('Deleted %s user groups') % len(_user_groups),
+                h.flash(_('Deleted %s user groups') % _user_groups,
                         category='success')
 
         def set_handle_flash_pull_requests():
             handle = handle_pull_requests
             if handle == 'detach':
-                h.flash(_('Detached %s pull requests') % len(_pull_requests),
+                h.flash(_('Detached %s pull requests') % _pull_requests,
                         category='success')
             elif handle == 'delete':
-                h.flash(_('Deleted %s pull requests') % len(_pull_requests),
+                h.flash(_('Deleted %s pull requests') % _pull_requests,
                         category='success')
 
         def set_handle_flash_artifacts():
             handle = handle_artifacts
             if handle == 'detach':
-                h.flash(_('Detached %s artifacts') % len(_artifacts),
+                h.flash(_('Detached %s artifacts') % _artifacts,
                         category='success')
             elif handle == 'delete':
-                h.flash(_('Deleted %s artifacts') % len(_artifacts),
+                h.flash(_('Deleted %s artifacts') % _artifacts,
                         category='success')
 
         handle_user = User.get_first_super_admin()

rhodecode/apps/gist/views.py

@@ -155,7 +155,6 b' class GistView(BaseAppView):'
             lifetime_options=[x[0] for x in c.lifetime_values])
 
         try:
-
             schema_data = schema.deserialize(data)
 
             # convert to safer format with just KEYs so we sure no duplicates
@@ -263,7 +262,7 b' class GistView(BaseAppView):'
             raise HTTPNotFound()
 
         if return_format == 'raw':
-            content = '\n\n'.join([f.content for f in c.files
+            content = b'\n\n'.join([f.content for f in c.files
                                    if (f_path is None or f.path == f_path)])
             response = Response(content)
             response.content_type = 'text/plain'

rhodecode/apps/home/views.py

@@ -70,7 +70,7 b' class HomeView(BaseAppView, DataGridAppV'
             if skip_default_user and usr['username'] == UserModel.cls.DEFAULT_USER:
                 return False
             return True
-        _users = filter(maybe_skip_default_user, _users)
+        _users = list(filter(maybe_skip_default_user, _users))
 
         if include_groups:
             # extend with user groups
@@ -229,7 +229,7 b' class HomeView(BaseAppView, DataGridAppV'
                 'value': org_query,
                 'value_display': 'user: `{}`'.format(obj.username),
                 'type': 'user',
-                'icon_link': h.gravatar_url(obj.email, 30),
+                'icon_link': h.gravatar_url(obj.email, 30, request=self.request),
                 'url': h.route_path(
                     'user_profile', username=obj.username)
             }

rhodecode/apps/repo_group/views/repo_group_settings.py

@@ -63,9 +63,8 b' class RepoGroupSettingsView(RepoGroupApp'
             show_empty_group=show_root_location)
         # filter out current repo group
         exclude_group_ids = [c.repo_group.group_id]
-        c.repo_groups = filter(lambda x: x[0] not in exclude_group_ids,
-                               c.repo_groups)
-        c.repo_groups_choices = map(lambda k: k[0], c.repo_groups)
+        c.repo_groups = [x for x in c.repo_groups if x[0] not in exclude_group_ids]
+        c.repo_groups_choices = [k[0] for k in c.repo_groups]
 
         parent_group = c.repo_group.parent_group
 
@@ -135,7 +134,7 b' class RepoGroupSettingsView(RepoGroupApp'
         schema = self._get_schema(c, old_values=old_values)
 
         c.form = RcForm(schema)
-        pstruct = self.request.POST.items()
+        pstruct = list(self.request.POST.items())
 
         try:
             schema_data = c.form.validate(pstruct)

rhodecode/apps/repository/__init__.py

@@ -990,14 +990,14 b' def includeme(config):'
         route_name='edit_repo_fields_delete', request_method='POST',
         renderer='rhodecode:templates/admin/repos/repo_edit.mako')
 
-    # Locking
+    # quick actions: locking
     config.add_route(
-        name='repo_edit_toggle_locking',
-        pattern='/{repo_name:.*?[^/]}/settings/toggle_locking', repo_route=True)
+        name='repo_settings_quick_actions',
+        pattern='/{repo_name:.*?[^/]}/settings/quick-action', repo_route=True)
     config.add_view(
         RepoSettingsView,
-        attr='edit_advanced_toggle_locking',
-        route_name='repo_edit_toggle_locking', request_method='GET',
+        attr='repo_settings_quick_actions',
+        route_name='repo_settings_quick_actions', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repo_edit.mako')
 
     # Remote

rhodecode/apps/repository/views/repo_commits.py

@@ -260,9 +260,10 b' class RepoCommitsView(RepoAppView):'
                         ignore_whitespace=hide_whitespace_changes,
                         context=diff_context)
 
-                    diff_processor = diffs.DiffProcessor(
-                        vcs_diff, format='newdiff', diff_limit=diff_limit,
-                        file_limit=file_limit, show_full_diff=c.fulldiff)
+                    diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
+                                                         diff_limit=diff_limit,
+                                                         file_limit=file_limit,
+                                                         show_full_diff=c.fulldiff)
 
                     _parsed = diff_processor.prepare()
 
@@ -285,8 +286,8 b' class RepoCommitsView(RepoAppView):'
                 _diff = self.rhodecode_vcs_repo.get_diff(
                     commit1, commit2,
                     ignore_whitespace=hide_whitespace_changes, context=diff_context)
-                diff_processor = diffs.DiffProcessor(
-                    _diff, format='newdiff', diff_limit=diff_limit,
+                diff_processor = diffs.DiffProcessor(_diff, diff_format='newdiff',
+                                                     diff_limit=diff_limit,
                     file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
@@ -643,17 +644,28 b' class RepoCommitsView(RepoAppView):'
 
         Session().commit()
 
-        return {
+        data = {
             'store_fid': store_uid,
             'access_path': h.route_path(
                 'download_file', fid=store_uid),
             'fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
+            # for EE those are replaced by FQN links on repo-only like
+            'repo_access_path': h.route_url(
+                'download_file', fid=store_uid),
+            'repo_fqn_access_path': h.route_url(
+                'download_file', fid=store_uid),
+        }
+        # this data is a part of CE/EE additional code
+        if c.rhodecode_edition_id == 'EE':
+            data.update({
             'repo_access_path': h.route_path(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
             'repo_fqn_access_path': h.route_url(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
-        }
+            })
+
+        return data
 
     @LoginRequired()
     @NotAnonymous()
@@ -766,7 +778,7 b' class RepoCommitsView(RepoAppView):'
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
-                'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
+                'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16, request=self.request),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }

rhodecode/apps/repository/views/repo_compare.py

@@ -282,9 +282,10 b' class RepoCompareView(RepoAppView):'
             path=target_path, path1=source_path,
             ignore_whitespace=hide_whitespace_changes, context=diff_context)
 
-        diff_processor = diffs.DiffProcessor(
-            txt_diff, format='newdiff', diff_limit=diff_limit,
-            file_limit=file_limit, show_full_diff=c.fulldiff)
+        diff_processor = diffs.DiffProcessor(txt_diff, diff_format='newdiff',
+                                             diff_limit=diff_limit,
+                                             file_limit=file_limit,
+                                             show_full_diff=c.fulldiff)
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(

rhodecode/apps/repository/views/repo_feed.py

@@ -17,8 +17,9 b''
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-import pytz
+
 import logging
+import datetime
 
 from pyramid.response import Response
 
@@ -72,8 +73,9 b' class RepoFeedView(RepoAppView):'
         self.feed_items_per_page = config['feed_items_per_page']
 
     def _changes(self, commit):
-        diff_processor = DiffProcessor(
-            commit.diff(), diff_limit=self.feed_diff_limit)
+        diff = commit.diff()
+        diff_processor = DiffProcessor(diff, diff_format='newdiff',
+                                       diff_limit=self.feed_diff_limit)
         _parsed = diff_processor.prepare(inline_diff=False)
         limited_diff = isinstance(_parsed, LimitedDiffContainer)
 
@@ -97,7 +99,7 b' class RepoFeedView(RepoAppView):'
             has_hidden_changes=has_hidden_changes
         )
 
-    def _set_timezone(self, date, tzinfo=pytz.utc):
+    def _set_timezone(self, date, tzinfo=datetime.timezone.utc):
         if not getattr(date, "tzinfo", None):
             date.replace(tzinfo=tzinfo)
         return date
@@ -114,7 +116,10 b' class RepoFeedView(RepoAppView):'
         return list(collection[-self.feed_items_per_page:])
 
     def uid(self, repo_id, commit_id):
-        return '{}:{}'.format(md5_safe(repo_id), md5_safe(commit_id))
+        return '{}:{}'.format(
+            md5_safe(repo_id, return_type='str'),
+            md5_safe(commit_id, return_type='str')
+        )
 
     @LoginRequired(auth_token_access=[UserApiKeys.ROLE_FEED])
     @HasRepoPermissionAnyDecorator(
@@ -126,7 +131,7 b' class RepoFeedView(RepoAppView):'
         self.load_default_context()
         force_recache = self.get_recache_flag()
 
-        cache_namespace_uid = 'cache_repo_feed.{}'.format(self.db_repo.repo_id)
+        cache_namespace_uid = 'repo_feed.{}'.format(self.db_repo.repo_id)
         condition = not (self.path_filter.is_enabled or force_recache)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
@@ -144,7 +149,7 b' class RepoFeedView(RepoAppView):'
             for commit in reversed(self._get_commits()):
                 date = self._set_timezone(commit.date)
                 feed.add_item(
-                    unique_id=self.uid(repo_id, commit.raw_id),
+                    unique_id=self.uid(str(repo_id), commit.raw_id),
                     title=self._get_title(commit),
                     author_name=commit.author,
                     description=self._get_description(commit),
@@ -173,7 +178,7 b' class RepoFeedView(RepoAppView):'
         self.load_default_context()
         force_recache = self.get_recache_flag()
 
-        cache_namespace_uid = 'cache_repo_feed.{}'.format(self.db_repo.repo_id)
+        cache_namespace_uid = 'repo_feed.{}'.format(self.db_repo.repo_id)
         condition = not (self.path_filter.is_enabled or force_recache)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
@@ -191,7 +196,7 b' class RepoFeedView(RepoAppView):'
             for commit in reversed(self._get_commits()):
                 date = self._set_timezone(commit.date)
                 feed.add_item(
-                    unique_id=self.uid(repo_id, commit.raw_id),
+                    unique_id=self.uid(str(repo_id), commit.raw_id),
                     title=self._get_title(commit),
                     author_name=commit.author,
                     description=self._get_description(commit),

rhodecode/apps/repository/views/repo_files.py

@@ -21,10 +21,10 b''
 import itertools
 import logging
 import os
-import shutil
-import tempfile
 import collections
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 import pathlib
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
@@ -38,12 +38,16 b' from rhodecode.apps._base import RepoApp'
 
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
+from rhodecode.lib.hash_utils import sha1_safe
+from rhodecode.lib.rc_cache.archive_cache import get_archival_cache_store, get_archival_config, ReentrantLock
+from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
 from rhodecode.lib.codeblocks import (
     filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
-from rhodecode.lib.utils2 import (
-    convert_line_endings, detect_mode, safe_str, str2bool, safe_int, sha1)
+from rhodecode.lib.utils2 import convert_line_endings, detect_mode
+from rhodecode.lib.type_utils import str2bool
+from rhodecode.lib.str_utils import safe_str, safe_int
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib.vcs import path as vcspath
@@ -61,6 +65,48 b' from rhodecode.model.db import Repositor'
 log = logging.getLogger(__name__)
 
 
+def get_archive_name(db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
+    # original backward compat name of archive
+    clean_name = safe_str(db_repo_name.replace('/', '_'))
+
+    # e.g vcsserver-sub-1-abcfdef-archive-all.zip
+    # vcsserver-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip
+
+    sub_repo = 'sub-1' if subrepos else 'sub-0'
+    commit = commit_sha if with_hash else 'archive'
+    path_marker = (path_sha if with_hash else '') or 'all'
+    archive_name = f'{clean_name}-{sub_repo}-{commit}-{path_marker}{ext}'
+
+    return archive_name
+
+
+def get_path_sha(at_path):
+    return safe_str(sha1_safe(at_path)[:8])
+
+
+def _get_archive_spec(fname):
+    log.debug('Detecting archive spec for: `%s`', fname)
+
+    fileformat = None
+    ext = None
+    content_type = None
+    for a_type, content_type, extension in settings.ARCHIVE_SPECS:
+
+        if fname.endswith(extension):
+            fileformat = a_type
+            log.debug('archive is of type: %s', fileformat)
+            ext = extension
+            break
+
+    if not fileformat:
+        raise ValueError()
+
+    # left over part of whole fname is the commit
+    commit_id = fname[:-len(ext)]
+
+    return commit_id, ext, fileformat, content_type
+
+
 class RepoFilesView(RepoAppView):
 
     @staticmethod
@@ -72,12 +118,12 b' class RepoFilesView(RepoAppView):'
         branches in the underlying repository.
         """
         tags_and_branches = itertools.chain(
-            repo.branches.iterkeys(),
-            repo.tags.iterkeys())
+            repo.branches.keys(),
+            repo.tags.keys())
         tags_and_branches = sorted(tags_and_branches, key=len, reverse=True)
 
         for name in tags_and_branches:
-            if f_path.startswith('{}/'.format(name)):
+            if f_path.startswith(f'{name}/'):
                 f_path = vcspath.relpath(f_path, name)
                 break
         return f_path
@@ -165,19 +211,23 b' class RepoFilesView(RepoAppView):'
             if not redirect_after:
                 return None
 
+            add_new = upload_new = ""
+            if h.HasRepoPermissionAny(
+                    'repository.write', 'repository.admin')(self.db_repo_name):
             _url = h.route_path(
                 'repo_files_add_file',
                 repo_name=self.db_repo_name, commit_id=0, f_path='')
+                add_new = h.link_to(
+                    _('add a new file'), _url, class_="alert-link")
 
-            if h.HasRepoPermissionAny(
-                    'repository.write', 'repository.admin')(self.db_repo_name):
-                add_new = h.link_to(
-                    _('Click here to add a new file.'), _url, class_="alert-link")
-            else:
-                add_new = ""
+                _url_upld = h.route_path(
+                    'repo_files_upload_file',
+                    repo_name=self.db_repo_name, commit_id=0, f_path='')
+                upload_new = h.link_to(
+                    _('upload a new file'), _url_upld, class_="alert-link")
 
             h.flash(h.literal(
-                _('There are no files yet. %s') % add_new), category='warning')
+                _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo_name))
 
@@ -189,7 +239,7 b' class RepoFilesView(RepoAppView):'
             h.flash(h.escape(safe_str(e)), category='error')
             raise HTTPNotFound()
 
-    def _get_filenode_or_redirect(self, commit_obj, path):
+    def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
@@ -197,7 +247,7 b' class RepoFilesView(RepoAppView):'
         _ = self.request.translate
 
         try:
-            file_node = commit_obj.get_node(path)
+            file_node = commit_obj.get_node(path, pre_load=pre_load)
             if file_node.is_dir():
                 raise RepositoryError('The given path is a directory')
         except CommitDoesNotExistError:
@@ -262,7 +312,7 b' class RepoFilesView(RepoAppView):'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
-        cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
+        cache_namespace_uid = 'repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
@@ -279,28 +329,6 b' class RepoFilesView(RepoAppView):'
             rc_cache.FILE_TREE_CACHE_VER, self.db_repo.repo_name_hash,
             self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
 
-    def _get_archive_spec(self, fname):
-        log.debug('Detecting archive spec for: `%s`', fname)
-
-        fileformat = None
-        ext = None
-        content_type = None
-        for a_type, content_type, extension in settings.ARCHIVE_SPECS:
-
-            if fname.endswith(extension):
-                fileformat = a_type
-                log.debug('archive is of type: %s', fileformat)
-                ext = extension
-                break
-
-        if not fileformat:
-            raise ValueError()
-
-        # left over part of whole fname is the commit
-        commit_id = fname[:-len(ext)]
-
-        return commit_id, ext, fileformat, content_type
-
     def create_pure_path(self, *parts):
         # Split paths and sanitize them, removing any ../ etc
         sanitized_path = [
@@ -325,22 +353,6 b' class RepoFilesView(RepoAppView):'
 
         return lf_enabled
 
-    def _get_archive_name(self, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
-        # original backward compat name of archive
-        clean_name = safe_str(db_repo_name.replace('/', '_'))
-
-        # e.g vcsserver.zip
-        # e.g vcsserver-abcdefgh.zip
-        # e.g vcsserver-abcdefgh-defghijk.zip
-        archive_name = '{}{}{}{}{}{}'.format(
-            clean_name,
-            '-sub' if subrepos else '',
-            commit_sha,
-            '-{}'.format('plain') if not with_hash else '',
-            '-{}'.format(path_sha) if path_sha else '',
-            ext)
-        return archive_name
-
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
@@ -360,7 +372,7 b' class RepoFilesView(RepoAppView):'
 
         try:
             commit_id, ext, fileformat, content_type = \
-                self._get_archive_spec(fname)
+                _get_archive_spec(fname)
         except ValueError:
             return Response(_('Unknown archive type for: `{}`').format(
                 h.escape(fname)))
@@ -383,69 +395,80 b' class RepoFilesView(RepoAppView):'
         except Exception:
             return Response(_('No node at path {} for this repository').format(h.escape(at_path)))
 
-        # path sha is part of subdir
-        path_sha = ''
-        if at_path != default_at_path:
-            path_sha = sha1(at_path)[:8]
-        short_sha = '-{}'.format(safe_str(commit.short_id))
-        # used for cache etc
-        archive_name = self._get_archive_name(
-            self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos,
-            path_sha=path_sha, with_hash=with_hash)
+        path_sha = get_path_sha(at_path)
+
+        # used for cache etc, consistent unique archive name
+        archive_name_key = get_archive_name(
+            self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
+            path_sha=path_sha, with_hash=True)
 
         if not with_hash:
-            short_sha = ''
             path_sha = ''
 
         # what end client gets served
-        response_archive_name = self._get_archive_name(
-            self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos,
+        response_archive_name = get_archive_name(
+            self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
             path_sha=path_sha, with_hash=with_hash)
+
         # remove extension from our archive directory name
         archive_dir_name = response_archive_name[:-len(ext)]
 
-        use_cached_archive = False
-        archive_cache_dir = CONFIG.get('archive_cache_dir')
-        archive_cache_enabled = archive_cache_dir and not self.request.GET.get('no_cache')
-        cached_archive_path = None
+        archive_cache_disable = self.request.GET.get('no_cache')
+
+        d_cache = get_archival_cache_store(config=CONFIG)
+        # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver
+        d_cache_conf = get_archival_config(config=CONFIG)
 
-        if archive_cache_enabled:
-            # check if we it's ok to write, and re-create the archive cache
-            if not os.path.isdir(CONFIG['archive_cache_dir']):
-                os.makedirs(CONFIG['archive_cache_dir'])
-
-            cached_archive_path = os.path.join(
-                CONFIG['archive_cache_dir'], archive_name)
-            if os.path.isfile(cached_archive_path):
-                log.debug('Found cached archive in %s', cached_archive_path)
-                fd, archive = None, cached_archive_path
+        reentrant_lock_key = archive_name_key + '.lock'
+        with ReentrantLock(d_cache, reentrant_lock_key):
+            # This is also a cache key
+            use_cached_archive = False
+            if archive_name_key in d_cache and not archive_cache_disable:
+                reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True)
                 use_cached_archive = True
+                log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',
+                          archive_name_key, tag, reader.name)
             else:
-                log.debug('Archive %s is not yet cached', archive_name)
+                reader = None
+                log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key)
 
         # generate new archive, as previous was not found in the cache
-        if not use_cached_archive:
-            _dir = os.path.abspath(archive_cache_dir) if archive_cache_dir else None
-            fd, archive = tempfile.mkstemp(dir=_dir)
-            log.debug('Creating new temp archive in %s', archive)
+            if not reader:
+                # first remove expired items, before generating a new one :)
+                # we di this manually because automatic eviction is disabled
+                d_cache.cull(retry=True)
+
             try:
-                commit.archive_repo(archive, archive_dir_name=archive_dir_name,
+                    commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
                                     kind=fileformat, subrepos=subrepos,
-                                    archive_at_path=at_path)
+                                        archive_at_path=at_path, cache_config=d_cache_conf)
             except ImproperArchiveTypeError:
                 return _('Unknown archive type')
-            if archive_cache_enabled:
-                # if we generated the archive and we have cache enabled
-                # let's use this for future
-                log.debug('Storing new archive in %s', cached_archive_path)
-                shutil.move(archive, cached_archive_path)
-                archive = cached_archive_path
+
+            reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True)
+
+        if not reader:
+            raise ValueError('archive cache reader is empty, failed to fetch file from distributed archive cache')
 
+        def archive_iterator(_reader):
+            while 1:
+                data = _reader.read(1024)
+                if not data:
+                    break
+                yield data
+
+        response = Response(app_iter=archive_iterator(reader))
+        response.content_disposition = f'attachment; filename={response_archive_name}'
+        response.content_type = str(content_type)
+
+        try:
+            return response
+        finally:
         # store download action
         audit_logger.store_web(
             'repo.archive.download', action_data={
                 'user_agent': self.request.user_agent,
-                'archive_name': archive_name,
+                    'archive_name': archive_name_key,
                 'archive_spec': fname,
                 'archive_cached': use_cached_archive},
             user=self._rhodecode_user,
@@ -453,33 +476,13 b' class RepoFilesView(RepoAppView):'
             commit=True
         )
 
-        def get_chunked_archive(archive_path):
-            with open(archive_path, 'rb') as stream:
-                while True:
-                    data = stream.read(16 * 1024)
-                    if not data:
-                        if fd:  # fd means we used temporary file
-                            os.close(fd)
-                        if not archive_cache_enabled:
-                            log.debug('Destroying temp archive %s', archive_path)
-                            os.remove(archive_path)
-                        break
-                    yield data
-
-        response = Response(app_iter=get_chunked_archive(archive))
-        response.content_disposition = str('attachment; filename=%s' % response_archive_name)
-        response.content_type = str(content_type)
-
-        return response
-
     def _get_file_node(self, commit_id, f_path):
         if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             try:
                 node = commit.get_node(f_path)
                 if node.is_dir():
-                    raise NodeError('%s path is a %s not a file'
-                                    % (node, type(node)))
+                    raise NodeError(f'{node} path is a {type(node)} not a file')
             except NodeDoesNotExistError:
                 commit = EmptyCommit(
                     commit_id=commit_id,
@@ -489,12 +492,12 b' class RepoFilesView(RepoAppView):'
                     message=commit.message,
                     author=commit.author,
                     date=commit.date)
-                node = FileNode(f_path, '', commit=commit)
+                node = FileNode(safe_bytes(f_path), b'', commit=commit)
         else:
             commit = EmptyCommit(
                 repo=self.rhodecode_vcs_repo,
                 alias=self.rhodecode_vcs_repo.alias)
-            node = FileNode(f_path, '', commit=commit)
+            node = FileNode(safe_bytes(f_path), b'', commit=commit)
         return node
 
     @LoginRequired()
@@ -551,12 +554,13 b' class RepoFilesView(RepoAppView):'
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
-            diff = diffs.DiffProcessor(_diff, format='gitdiff')
+            # NOTE: this was using diff_format='gitdiff'
+            diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             response.content_disposition = (
-                'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2)
+                f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff'
             )
             charset = self._get_default_encoding(c)
             if charset:
@@ -567,7 +571,8 b' class RepoFilesView(RepoAppView):'
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
-            diff = diffs.DiffProcessor(_diff, format='gitdiff')
+            # NOTE: this was using diff_format='gitdiff'
+            diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
@@ -637,8 +642,7 b' class RepoFilesView(RepoAppView):'
         c.annotate = view_name == 'repo_files:annotated'
         # default is false, but .rst/.md files later are auto rendered, we can
         # overwrite auto rendering by setting this GET flag
-        c.renderer = view_name == 'repo_files:rendered' or \
-                        not self.request.GET.get('no-render', False)
+        c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False)
 
         commit_id, f_path = self._get_commit_and_path()
 
@@ -675,7 +679,7 b' class RepoFilesView(RepoAppView):'
 
         # files or dirs
         try:
-            c.file = c.commit.get_node(f_path)
+            c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data'])
 
             c.file_author = True
             c.file_tree = ''
@@ -917,7 +921,7 b' class RepoFilesView(RepoAppView):'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
-        cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
+        cache_namespace_uid = 'repo.{}'.format(repo_id)
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
@@ -950,7 +954,7 b' class RepoFilesView(RepoAppView):'
 
         metadata = self._get_nodelist_at_commit(
             self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
-        return {'nodes': metadata}
+        return {'nodes': [x for x in metadata]}
 
     def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type):
         items = []
@@ -967,7 +971,7 b' class RepoFilesView(RepoAppView):'
 
         # NOTE(dan): old code we used in "diff" mode compare
         new_f_path = vcspath.join(name, f_path)
-        return u'%s@%s' % (new_f_path, commit_id)
+        return f'{new_f_path}@{commit_id}'
 
     def _get_node_history(self, commit_obj, f_path, commits=None):
         """
@@ -1194,8 +1198,8 b' class RepoFilesView(RepoAppView):'
         message = self.request.POST.get('message') or c.default_message
         try:
             nodes = {
-                node_path: {
-                    'content': ''
+                safe_bytes(node_path): {
+                    'content': b''
                 }
             }
             ScmModel().delete_nodes(
@@ -1273,7 +1277,7 b' class RepoFilesView(RepoAppView):'
         c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
-        old_content = c.file.content
+        old_content = c.file.str_content
         sl = old_content.splitlines(1)
         first_line = sl[0] if sl else ''
 
@@ -1283,7 +1287,8 b' class RepoFilesView(RepoAppView):'
         content = convert_line_endings(r_post.get('content', ''), line_ending_mode)
 
         message = r_post.get('message') or c.default_message
-        org_node_path = c.file.unicode_path
+
+        org_node_path = c.file.str_path
         filename = r_post['filename']
 
         root_path = c.file.dir_path
@@ -1299,10 +1304,10 b' class RepoFilesView(RepoAppView):'
 
         try:
             mapping = {
-                org_node_path: {
+                c.file.bytes_path: {
                     'org_filename': org_node_path,
-                    'filename': node_path,
-                    'content': content,
+                    'filename': safe_bytes(node_path),
+                    'content': safe_bytes(content),
                     'lexer': '',
                     'op': 'mod',
                     'mode': c.file.mode
@@ -1400,8 +1405,6 b' class RepoFilesView(RepoAppView):'
         filename = r_post.get('filename')
         unix_mode = 0
 
-        content = convert_line_endings(r_post.get('content', ''), unix_mode)
-
         if not filename:
             # If there's no commit, redirect to repo summary
             if type(c.commit) is EmptyCommit:
@@ -1417,9 +1420,10 b' class RepoFilesView(RepoAppView):'
         node_path = pure_path.as_posix().lstrip('/')
 
         author = self._rhodecode_db_user.full_contact
+        content = convert_line_endings(r_post.get('content', ''), unix_mode)
         nodes = {
-            node_path: {
-                'content': content
+            safe_bytes(node_path): {
+                'content': safe_bytes(content)
             }
         }
 
@@ -1518,7 +1522,7 b' class RepoFilesView(RepoAppView):'
             pure_path = self.create_pure_path(root_path, filename)
             node_path = pure_path.as_posix().lstrip('/')
 
-            nodes[node_path] = {
+            nodes[safe_bytes(node_path)] = {
                 'content': content
             }
 

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -234,8 +234,7 b' class RepoPullRequestsView(RepoAppView, '
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
-        diff_processor = diffs.DiffProcessor(
-            vcs_diff, format='newdiff', diff_limit=diff_limit,
+        diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
@@ -259,8 +258,8 b' class RepoPullRequestsView(RepoAppView, '
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
-        diff_processor = diffs.DiffProcessor(
-            vcs_diff, format='newdiff', diff_limit=diff_limit,
+        diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
+                                             diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
@@ -933,7 +932,7 b' class RepoPullRequestsView(RepoAppView, '
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
-                ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
+                ilike_expression = u'%{}%'.format(safe_str(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()

rhodecode/apps/repository/views/repo_settings.py

@@ -249,3 +249,29 b' class RepoSettingsView(RepoAppView):'
                     category='error')
         raise HTTPFound(
             h.route_path('edit_repo_statistics', repo_name=self.db_repo_name))
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.admin')
+    def repo_settings_quick_actions(self):
+        _ = self.request.translate
+
+        set_lock = self.request.GET.get('set_lock')
+        set_unlock = self.request.GET.get('set_unlock')
+
+        try:
+            if set_lock:
+                Repository.lock(self.db_repo, self._rhodecode_user.user_id,
+                                lock_reason=Repository.LOCK_WEB)
+                h.flash(_('Locked repository'), category='success')
+            elif set_unlock:
+                Repository.unlock(self.db_repo)
+                h.flash(_('Unlocked repository'), category='success')
+        except Exception as e:
+            log.exception("Exception during unlocking")
+            h.flash(_('An error occurred during unlocking'), category='error')
+
+        raise HTTPFound(
+            h.route_path('repo_summary', repo_name=self.db_repo_name))
+
+
+

rhodecode/apps/ssh_support/lib/backends/__init__.py

@@ -63,23 +63,23 b' class SshWrapper(object):'
         from rhodecode.model.meta import raw_query_executor, Base
 
         table = Table('user_ssh_keys', Base.metadata, autoload=False)
+        atime = datetime.datetime.utcnow()
         stmt = (
             table.update()
             .where(table.c.ssh_key_id == key_id)
-            .values(accessed_on=datetime.datetime.utcnow())
-            .returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
+            .values(accessed_on=atime)
+            # no MySQL Support for .returning :((
+            #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
         )
 
-        scalar_res = None
+        res_count = None
         with raw_query_executor() as session:
             result = session.execute(stmt)
             if result.rowcount:
-                scalar_res = result.first()
+                res_count = result.rowcount
 
-        if scalar_res:
-            atime, ssh_key_fingerprint = scalar_res
-            log.debug('Update key id:`%s` fingerprint:`%s` access time',
-                      key_id, ssh_key_fingerprint)
+        if res_count:
+            log.debug('Update key id:`%s` access time', key_id)
 
     def get_user(self, user_id):
         user = AttributeDict()

rhodecode/apps/ssh_support/utils.py

@@ -40,8 +40,8 b" SSH_OPTS = 'no-pty,no-port-forwarding,no"
 
 def get_all_active_keys():
     result = UserSshKeys.query() \
-        .options(joinedload(UserSshKeys.user)) \
-        .filter(UserSshKeys.user != User.get_default_user()) \
+        .join(User) \
+        .filter(User != User.get_default_user()) \
         .filter(User.active == true()) \
         .all()
     return result
@@ -55,6 +55,10 b' def _generate_ssh_authorized_keys_file('
         os.path.expanduser(authorized_keys_file_path))
     tmp_file_dir = os.path.dirname(authorized_keys_file_path)
 
+    if not os.path.exists(tmp_file_dir):
+        log.debug('SSH authorized_keys file dir does not exist, creating one now...')
+        os.makedirs(tmp_file_dir)
+
     all_active_keys = get_all_active_keys()
 
     if allow_shell:
@@ -65,6 +69,7 b' def _generate_ssh_authorized_keys_file('
     if not os.path.isfile(authorized_keys_file_path):
         log.debug('Creating file at %s', authorized_keys_file_path)
         with open(authorized_keys_file_path, 'w'):
+            # create a file with write access
             pass
 
     if not os.access(authorized_keys_file_path, os.R_OK):
@@ -78,7 +83,7 b' def _generate_ssh_authorized_keys_file('
         dir=tmp_file_dir)
 
     now = datetime.datetime.utcnow().isoformat()
-    keys_file = os.fdopen(fd, 'wb')
+    keys_file = os.fdopen(fd, 'wt')
     keys_file.write(HEADER.format(len(all_active_keys), now))
     ini_path = rhodecode.CONFIG['__file__']
 

rhodecode/apps/user_group/views/__init__.py

@@ -75,7 +75,7 b' class UserGroupsView(UserGroupAppView):'
                 'first_name': user.first_name,
                 'last_name': user.last_name,
                 'username': user.username,
-                'icon_link': h.gravatar_url(user.email, 30),
+                'icon_link': h.gravatar_url(user.email, 30, request=self.request),
                 'value_display': h.person(user.email),
                 'value': user.username,
                 'value_type': 'user',