app: new optimized remote endpoints for python3 rewrite

vcsserver/remote/git.py

@@ -1,1382 +1,1463 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import collections
 import logging
 import os
-import posixpath as vcspath
 import re
 import stat
 import traceback
 import urllib.request
 import urllib.parse
 import urllib.error
 from functools import wraps
 
 import more_itertools
 import pygit2
 from pygit2 import Repository as LibGit2Repo
 from pygit2 import index as LibGit2Index
 from dulwich import index, objects
-from dulwich.client import HttpGitClient, LocalGitClient
+from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes
-from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo, BinaryEnvelope
+from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
 from vcsserver.vcs_base import RemoteBase
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 PEELED_REF_MARKER = b'^{}'
 HEAD_MARKER = b'HEAD'
 
 log = logging.getLogger(__name__)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
 
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
             exc = exceptions.LookupException(org_exc=e)
             raise exc(safe_str(e))
         except (HangupException, UnexpectedCommandError) as e:
             exc = exceptions.VcsException(org_exc=e)
             raise exc(safe_str(e))
         except Exception:
             # NOTE(marcink): because of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
                 #log.exception("Unhandled exception in git remote call")
                 #raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class Repository(LibGit2Repo):
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.free()
 
 
 class GitFactory(RepoFactory):
     repo_type = 'git'
 
     def _create_repo(self, wire, create, use_libgit2=False):
         if use_libgit2:
             repo = Repository(safe_bytes(wire['path']))
         else:
             # dulwich mode
             repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
             repo = Repo(repo_path)
 
         log.debug('repository created: got GIT object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, use_libgit2=False):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create, use_libgit2)
 
     def repo_libgit2(self, wire):
         return self.repo(wire, use_libgit2=True)
 
 
+def create_signature_from_string(author_str, **kwargs):
+    """
+    Creates a pygit2.Signature object from a string of the format 'Name <email>'.
+
+    :param author_str: String of the format 'Name <email>'
+    :return: pygit2.Signature object
+    """
+    match = re.match(r'^(.+) <(.+)>$', author_str)
+    if match is None:
+        raise ValueError(f"Invalid format: {author_str}")
+
+    name, email = match.groups()
+    return pygit2.Signature(name, email, **kwargs)
+
+
+def get_obfuscated_url(url_obj):
+    url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
+    url_obj.query = obfuscate_qs(url_obj.query)
+    obfuscated_uri = str(url_obj)
+    return obfuscated_uri
+
+
 class GitRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
         self._bulk_methods = {
             "date": self.date,
             "author": self.author,
             "branch": self.branch,
             "message": self.message,
             "parents": self.parents,
             "_commit": self.revision,
         }
+        self._bulk_file_methods = {
+            "size": self.get_node_size,
+            "data": self.get_node_data,
+            "flags": self.get_node_flags,
+            "is_binary": self.get_node_is_binary,
+            "md5": self.md5_hash
+        }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
             return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
         return {}
 
     def _remote_conf(self, config):
         params = [
             '-c', 'core.askpass=""',
         ]
         ssl_cert_dir = config.get('vcs_ssl_dir')
         if ssl_cert_dir:
             params.extend(['-c', f'http.sslCAinfo={ssl_cert_dir}'])
         return params
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = b'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return safe_str(stdout.strip())
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
 
             try:
                 has_head = repo.head.name
                 if has_head:
                     return False
 
                 # NOTE(marcink): check again using more expensive method
                 return repo.is_empty
             except Exception:
                 pass
 
             return True
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id, fast_check):
             if fast_check:
                 path = safe_str(wire['path'])
                 if pygit2.discover_repository(path):
                     return True
                 return False
             else:
                 try:
                     repo_init = self._factory.repo_libgit2(wire)
                     with repo_init:
                         pass
                 except pygit2.GitError:
                     path = wire.get('path')
                     tb = traceback.format_exc()
                     log.debug("Invalid Git path `%s`, tb: %s", path, tb)
                     return False
                 return True
 
         return _assert_correct_path(context_uid, repo_id, True)
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             return repo.is_bare
 
     @reraise_safe_exceptions
+    def get_node_data(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return BytesEnvelope(blob_obj.data)
+
+    @reraise_safe_exceptions
+    def get_node_size(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.size
+
+    @reraise_safe_exceptions
+    def get_node_flags(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.filemode
+
+    @reraise_safe_exceptions
+    def get_node_is_binary(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.is_binary
+
+    @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob_obj = repo[sha]
-            return BinaryEnvelope(blob_obj.data)
+            return BytesEnvelope(blob_obj.data)
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _blob_raw_length(_repo_id, _sha):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[sha]
                 return blob.size
 
         return _blob_raw_length(repo_id, sha)
 
     def _parse_lfs_pointer(self, raw_content):
         spec_string = b'version https://git-lfs.github.com/spec'
         if raw_content and raw_content.startswith(spec_string):
 
             pattern = re.compile(rb"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
             ^size[ ](?P<oid_size>[0-9]+)\n
             (?:\n)?
             """, re.VERBOSE | re.MULTILINE)
             match = pattern.match(raw_content)
             if match:
                 return match.groupdict()
 
         return {}
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[commit_id]
                 if blob.is_binary:
                     return {}
 
                 return self._parse_lfs_pointer(blob.data)
 
         return _is_large_file(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _tree_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob_obj = repo[tree_id]
                 return blob_obj.is_binary
 
         return _is_binary(repo_id, tree_id)
 
     @reraise_safe_exceptions
-    def md5_hash(self, wire, tree_id):
+    def md5_hash(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _md5_hash(_repo_id, _tree_id):
+        def _md5_hash(_repo_id, _commit_id, _path):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[_commit_id]
+                blob_obj = commit.tree[_path]
+
+                if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                    raise exceptions.LookupException()(
+                        f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
+
             return ''
 
-        return _md5_hash(repo_id, tree_id)
+        return _md5_hash(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
 
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.has_oid()
 
         return False
 
     @reraise_safe_exceptions
     def store_path(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
         raise ValueError(f'Unable to fetch oid with path {oid}')
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _rev, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
                     wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
                     args = [wire, rev]
                     result[attr] = method(*args)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
             return result
 
         return _bulk_request(repo_id, rev, sorted(pre_load))
 
-    def _build_opener(self, url):
+    @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
+
+    def _build_opener(self, url: str):
         handlers = []
-        url_obj = url_parser(url)
-        _, authinfo = url_obj.authinfo()
+        url_obj = url_parser(safe_bytes(url))
+        authinfo = url_obj.authinfo()[1]
 
         if authinfo:
             # create a password manager
             passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib.request.build_opener(*handlers)
 
-    def _type_id_to_name(self, type_id: int):
-        return {
-            1: 'commit',
-            2: 'tree',
-            3: 'blob',
-            4: 'tag'
-        }[type_id]
-
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(safe_bytes(url))
-        test_uri, _ = url_obj.authinfo()
-        url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
-        url_obj.query = obfuscate_qs(url_obj.query)
-        cleaned_uri = str(url_obj)
-        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
+
+        test_uri = safe_str(url_obj.authinfo()[0])
+        obfuscated_uri = get_obfuscated_url(url_obj)
+
+        log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
-        o = self._build_opener(url)
+        o = self._build_opener(test_uri)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.parse.urlencode(q)
         cu = "{}{}".format(test_uri, qs)
         req = urllib.request.Request(cu, None, {})
 
         try:
-            log.debug("Trying to open URL %s", cleaned_uri)
+            log.debug("Trying to open URL %s", obfuscated_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
-            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
+            log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
             # means it cannot be cloned
-            raise exceptions.URLError(e)("[{}] org_exc: {}".format(cleaned_uri, e))
+            raise exceptions.URLError(e)("[{}] org_exc: {}".format(obfuscated_uri, e))
 
         # now detect if it's proper git repo
-        gitdata = resp.read()
-        if 'service=git-upload-pack' in gitdata:
+        gitdata: bytes = resp.read()
+
+        if b'service=git-upload-pack' in gitdata:
             pass
-        elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
+        elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
-            raise exceptions.URLError()(
-                "url [{}] does not look like an git".format(cleaned_uri))
+            e = None
+            raise exceptions.URLError(e)(
+                "url [%s] does not look like an hg repo org_exc: %s"
+                % (obfuscated_uri, e))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
         remote_refs = self.pull(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     @reraise_safe_exceptions
     def branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _branch(_context_uid, _repo_id, _commit_id):
             regex = re.compile('^refs/heads')
 
             def filter_with(ref):
                 return regex.match(ref[0]) and ref[1] == _commit_id
 
             branches = list(filter(filter_with, list(self.get_refs(wire).items())))
             return [x[0].split('refs/heads/')[-1] for x in branches]
 
         return _branch(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def commit_branches(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _commit_branches(_context_uid, _repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 branches = [x for x in repo.branches.with_commit(_commit_id)]
                 return branches
 
         return _commit_branches(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob = objects.Blob()
             blob.set_raw_string(content)
             repo.object_store.add_object(blob)
             return blob.id
 
-    # TODO: this is quite complex, check if that can be simplified
+    @reraise_safe_exceptions
+    def create_commit(self, wire, author, committer, message, branch, new_tree_id, date_args: list[int, int] = None):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+
+            if date_args:
+                current_time, offset = date_args
+
+                kw = {
+                    'time': current_time,
+                    'offset': offset
+                }
+            author = create_signature_from_string(author, **kw)
+            committer = create_signature_from_string(committer, **kw)
+
+            tree = new_tree_id
+            if isinstance(tree, (bytes, str)):
+                # validate this tree is in the repo...
+                tree = repo[safe_str(tree)].id
+
+            parents = []
+            # ensure we COMMIT on top of given branch head
+            # check if this repo has ANY branches, otherwise it's a new branch case we need to make
+            if branch in repo.branches.local:
+                parents += [repo.branches[branch].target]
+            elif [x for x in repo.branches.local]:
+                parents += [repo.head.target]
+            #else:
+                # in case we want to commit on new branch we create it on top of HEAD
+                #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
+
+            # # Create a new commit
+            commit_oid = repo.create_commit(
+                f'refs/heads/{branch}',  # the name of the reference to update
+                author,  # the author of the commit
+                committer,  # the committer of the commit
+                message,  # the commit message
+                tree,  # the tree produced by the index
+                parents  # list of parents for the new commit, usually just one,
+            )
+
+            new_commit_id = safe_str(commit_oid)
+
+            return new_commit_id
+
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
-        # Defines the root tree
-        class _Root(object):
-            def __repr__(self):
-                return 'ROOT TREE'
-        ROOT = _Root()
 
-        repo = self._factory.repo(wire)
-        object_store = repo.object_store
-
-        # Create tree and populates it with blobs
-        if commit_tree:
-            commit_tree = safe_bytes(commit_tree)
-
-        if commit_tree and repo[commit_tree]:
-            git_commit = repo[safe_bytes(commit_data['parents'][0])]
-            commit_tree = repo[git_commit.tree]  # root tree
-        else:
-            commit_tree = objects.Tree()
-
-        for node in updated:
-            # Compute subdirs if needed
-            dirpath, nodename = vcspath.split(node['path'])
-            dirnames = list(map(safe_str, dirpath and dirpath.split('/') or []))
-            parent = commit_tree
-            ancestors = [('', parent)]
+        def mode2pygit(mode):
+            """
+            git only supports two filemode 644 and 755
 
-            # Tries to dig for the deepest existing tree
-            while dirnames:
-                curdir = dirnames.pop(0)
-                try:
-                    dir_id = parent[curdir][1]
-                except KeyError:
-                    # put curdir back into dirnames and stops
-                    dirnames.insert(0, curdir)
-                    break
-                else:
-                    # If found, updates parent
-                    parent = repo[dir_id]
-                    ancestors.append((curdir, parent))
-            # Now parent is deepest existing tree and we need to create
-            # subtrees for dirnames (in reverse order)
-            # [this only applies for nodes from added]
-            new_trees = []
+            0o100755 -> 33261
+            0o100644 -> 33188
+            """
+            return {
+                0o100644: pygit2.GIT_FILEMODE_BLOB,
+                0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
+                0o120000: pygit2.GIT_FILEMODE_LINK
+            }.get(mode) or pygit2.GIT_FILEMODE_BLOB
 
-            blob = objects.Blob.from_string(node['content'])
-
-            node_path = safe_bytes(node['node_path'])
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo_index = repo.index
 
-            if dirnames:
-                # If there are trees which should be created we need to build
-                # them now (in reverse order)
-                reversed_dirnames = list(reversed(dirnames))
-                curtree = objects.Tree()
-                curtree[node_path] = node['mode'], blob.id
-                new_trees.append(curtree)
-                for dirname in reversed_dirnames[:-1]:
-                    newtree = objects.Tree()
-                    newtree[dirname] = (DIR_STAT, curtree.id)
-                    new_trees.append(newtree)
-                    curtree = newtree
-                parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
-            else:
-                parent.add(name=node_path, mode=node['mode'], hexsha=blob.id)
+            for pathspec in updated:
+                blob_id = repo.create_blob(pathspec['content'])
+                ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
+                repo_index.add(ie)
 
-            new_trees.append(parent)
-            # Update ancestors
-            reversed_ancestors = reversed(
-                [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
-            for parent, tree, path in reversed_ancestors:
-                parent[path] = (DIR_STAT, tree.id)
-                object_store.add_object(tree)
+            for pathspec in removed:
+                repo_index.remove(pathspec)
 
-            object_store.add_object(blob)
-            for tree in new_trees:
-                object_store.add_object(tree)
+            # Write changes to the index
+            repo_index.write()
+
+            # Create a tree from the updated index
+            commit_tree = repo_index.write_tree()
+
+        new_tree_id = commit_tree
 
-        for node_path in removed:
-            paths = node_path.split('/')
-            tree = commit_tree  # start with top-level
-            trees = [{'tree': tree, 'path': ROOT}]
-            # Traverse deep into the forest...
-            # resolve final tree by iterating the path.
-            # e.g a/b/c.txt will get
-            # - root as tree then
-            # - 'a' as tree,
-            # - 'b' as tree,
-            # - stop at c as blob.
-            for path in paths:
-                try:
-                    obj = repo[tree[path][1]]
-                    if isinstance(obj, objects.Tree):
-                        trees.append({'tree': obj, 'path': path})
-                        tree = obj
-                except KeyError:
-                    break
-            #PROBLEM:
-            """
-            We're not editing same reference tree object
-            """
-            # Cut down the blob and all rotten trees on the way back...
-            for path, tree_data in reversed(list(zip(paths, trees))):
-                tree = tree_data['tree']
-                tree.__delitem__(path)
-                # This operation edits the tree, we need to mark new commit back
+        author = commit_data['author']
+        committer = commit_data['committer']
+        message = commit_data['message']
+
+        date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
 
-                if len(tree) > 0:
-                    # This tree still has elements - don't remove it or any
-                    # of it's parents
-                    break
-
-        object_store.add_object(commit_tree)
+        new_commit_id = self.create_commit(wire, author, committer, message, branch,
+                                           new_tree_id, date_args=date_args)
 
-        # Create commit
-        commit = objects.Commit()
-        commit.tree = commit_tree.id
-        bytes_keys = [
-            'author',
-            'committer',
-            'message',
-            'encoding',
-            'parents'
-        ]
+        # libgit2, ensure the branch is there and exists
+        self.create_branch(wire, branch, new_commit_id)
 
-        for k, v in commit_data.items():
-            if k in bytes_keys:
-                if k == 'parents':
-                    v = [safe_bytes(x) for x in v]
-                else:
-                    v = safe_bytes(v)
-            setattr(commit, k, v)
+        # libgit2, set new ref to this created commit
+        self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
 
-        object_store.add_object(commit)
-
-        self.create_branch(wire, branch, safe_str(commit.id))
-
-        # dulwich set-ref
-        repo.refs[safe_bytes(f'refs/heads/{branch}')] = commit.id
-
-        return commit.id
+        return new_commit_id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
-            url_obj = url_parser(url)
+            url_obj = url_parser(safe_bytes(url))
             o = self._build_opener(url)
-            url, _ = url_obj.authinfo()
+            url = url_obj.authinfo()[0]
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             refs = [ascii_bytes(x) for x in refs]
 
             def determine_wants_requested(remote_refs):
                 determined = []
                 for ref_name, ref_hash in remote_refs.items():
                     bytes_ref_name = safe_bytes(ref_name)
 
                     if bytes_ref_name in refs:
                         bytes_ref_hash = safe_bytes(ref_hash)
                         determined.append(bytes_ref_hash)
                 return determined
 
             # swap with our custom requested wants
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
 
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             for k in remote_refs:
                 if k.endswith(PEELED_REF_MARKER):
                     log.debug("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
                 repo[HEAD_MARKER] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to check out HEAD
             repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo[HEAD_MARKER].tree)
+
+        if isinstance(remote_refs, FetchPackResult):
+            return remote_refs.refs
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_fetch(self, wire, url, refs=None, all_refs=False):
         self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
 
         config = self._wire_to_config(wire)
         # get all remote refs we'll use to fetch later
         cmd = ['ls-remote']
         if not all_refs:
             cmd += ['--heads', '--tags']
         cmd += [url]
         output, __ = self.run_git_command(
             wire, cmd, fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         remote_refs = collections.OrderedDict()
         fetch_refs = []
 
         for ref_line in output.splitlines():
             sha, ref = ref_line.split(b'\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
                 continue
             if ref.endswith(PEELED_REF_MARKER):
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
             if ref in [HEAD_MARKER]:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
                 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
             elif not refs:
                 fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
         log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
 
         if fetch_refs:
             for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
                 fetch_refs_chunks = list(chunk)
                 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
                 self.run_git_command(
                     wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
                     extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url, refs=None):
         if not self.check_url(url, wire):
             return
         config = self._wire_to_config(wire)
         self._factory.repo(wire)
         self.run_git_command(
             wire, ['push', url, '--mirror'], fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
-        LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
+        LocalGitClient(thin_packs=False).fetch(path2, repo_remote)
 
         revs = [
             x.commit.id
-            for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
+            for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha, maybe_unreachable=False):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_object(_context_uid, _repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
 
                 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
                 try:
                     commit = repo.revparse_single(sha)
                 except KeyError:
                     # NOTE(marcink): KeyError doesn't give us any meaningful information
                     # here, we instead give something more explicit
                     e = exceptions.RefNotFoundException('SHA: %s not found', sha)
                     raise exceptions.LookupException(e)(missing_commit_err)
                 except ValueError as e:
                     raise exceptions.LookupException(e)(missing_commit_err)
 
                 is_tag = False
                 if isinstance(commit, pygit2.Tag):
                     commit = repo.get(commit.target)
                     is_tag = True
 
                 check_dangling = True
                 if is_tag:
                     check_dangling = False
 
                 if check_dangling and maybe_unreachable:
                     check_dangling = False
 
                 # we used a reference and it parsed means we're not having a dangling commit
                 if sha != commit.hex:
                     check_dangling = False
 
                 if check_dangling:
                     # check for dangling commit
                     for branch in repo.branches.with_commit(commit.hex):
                         if branch:
                             break
                     else:
                         # NOTE(marcink): Empty error doesn't give us any meaningful information
                         # here, we instead give something more explicit
                         e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
                         raise exceptions.LookupException(e)(missing_commit_err)
 
                 commit_id = commit.hex
-                type_id = commit.type
+                type_str = commit.type_str
 
                 return {
                     'id': commit_id,
-                    'type': self._type_id_to_name(type_id),
+                    'type': type_str,
                     'commit_id': commit_id,
                     'idx': 0
                 }
 
         return _get_object(context_uid, repo_id, sha)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_refs(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 regex = re.compile('^refs/(heads|tags)/')
                 return {x.name: x.target.hex for x in
                         [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
 
         return _get_refs(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def get_branch_pointers(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_branch_pointers(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             regex = re.compile('^refs/heads')
             with repo_init as repo:
                 branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
                 return {x.target.hex: x.shorthand for x in branches}
 
         return _get_branch_pointers(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def head(self, wire, show_exc=True):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _head(_context_uid, _repo_id, _show_exc):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     return repo.head.peel().hex
                 except Exception:
                     if show_exc:
                         raise
         return _head(context_uid, repo_id, show_exc)
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = safe_str(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = safe_str(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _revision(_context_uid, _repo_id, _rev):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[rev]
                 obj_data = {
                     'id': commit.id.hex,
                 }
                 # tree objects itself don't have tree_id attribute
                 if hasattr(commit, 'tree_id'):
                     obj_data['tree'] = commit.tree_id.hex
 
                 return obj_data
         return _revision(context_uid, repo_id, rev)
 
     @reraise_safe_exceptions
     def date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _date(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'commit_time'):
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
                 else:
                     commit = commit.get_object()
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
 
                 # TODO(marcink): check dulwich difference of offset vs timezone
                 return [commit_time, commit_time_offset]
         return _date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def author(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _author(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'author'):
                     author = commit.author
                 else:
                     author = commit.get_object().author
 
                 if author.email:
                     return f"{author.name} <{author.email}>"
 
                 try:
                     return f"{author.name}"
                 except Exception:
                     return f"{safe_str(author.raw_name)}"
 
         return _author(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def message(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _message(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 return commit.message
         return _message(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _parents(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 if hasattr(commit, 'parent_ids'):
                     parent_ids = commit.parent_ids
                 else:
                     parent_ids = commit.get_object().parent_ids
 
                 return [x.hex for x in parent_ids]
         return _parents(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         head = self.head(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _children(_repo_id, _commit_id):
 
             output, __ = self.run_git_command(
                 wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
 
             child_ids = []
             pat = re.compile(fr'^{commit_id}')
             for line in output.splitlines():
                 line = safe_str(line)
                 if pat.match(line):
                     found_ids = line.split(' ')[1:]
                     child_ids.extend(found_ids)
                     break
 
             return child_ids
         return _children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.create(key, value, force=True)
 
     @reraise_safe_exceptions
     def create_branch(self, wire, branch_name, commit_id, force=False):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
+            if commit_id:
             commit = repo[commit_id]
+            else:
+                # if commit is not given  just use the HEAD
+                commit = repo.head()
 
             if force:
                 repo.branches.local.create(branch_name, commit, force=force)
             elif not repo.branches.get(branch_name):
                 # create only if that branch isn't existing
                 repo.branches.local.create(branch_name, commit, force=force)
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tag_remove(self, wire, tag_name):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             key = f'refs/tags/{tag_name}'
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
-        # TODO(marcink): remove this seems it's only used by tests
         repo = self._factory.repo(wire)
+        # source can be empty
+        source_id = safe_bytes(source_id if source_id else b'')
+        target_id = safe_bytes(target_id)
+
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
-        return list(result)
+
+        added = set()
+        modified = set()
+        deleted = set()
+        for (old_path, new_path), (_, _), (_, _) in list(result):
+            if new_path and old_path:
+                modified.add(new_path)
+            elif new_path and not old_path:
+                added.add(new_path)
+            elif not new_path and old_path:
+                deleted.add(old_path)
+
+        return list(added), list(modified), list(deleted)
 
     @reraise_safe_exceptions
     def tree_and_type_for_path(self, wire, commit_id, path):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
             repo_init = self._factory.repo_libgit2(wire)
 
             with repo_init as repo:
                 commit = repo[commit_id]
                 try:
                     tree = commit.tree[path]
                 except KeyError:
                     return None, None, None
 
                 return tree.id.hex, tree.type_str, tree.filemode
         return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_items(_repo_id, _tree_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     tree = repo[tree_id]
                 except KeyError:
                     raise ObjectMissing(f'No tree with id: {tree_id}')
 
                 result = []
                 for item in tree:
                     item_sha = item.hex
                     item_mode = item.filemode
                     item_type = item.type_str
 
                     if item_type == 'commit':
                         # NOTE(marcink): submodules we translate to 'link' for backward compat
                         item_type = 'link'
 
                     result.append((item.name, item_mode, item_sha, item_type))
                 return result
         return _tree_items(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         """
         Old version that uses subprocess to call diff
         """
 
         flags = [
             '-U%s' % context, '--patch',
             '--binary',
             '--find-renames',
             '--no-indent-heuristic',
             # '--indent-heuristic',
             #'--full-index',
             #'--abbrev=40'
         ]
 
         if opt_ignorews:
             flags.append('--ignore-all-space')
 
         if commit_id_1 == self.EMPTY_COMMIT:
             cmd = ['show'] + flags + [commit_id_2]
         else:
             cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
 
         if file_filter:
             cmd.extend(['--', file_filter])
 
         diff, __ = self.run_git_command(wire, cmd)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if commit_id_1 == self.EMPTY_COMMIT:
             lines = diff.splitlines()
             x = 0
             for line in lines:
                 if line.startswith(b'diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
             diff = '\n'.join(lines[x:]) + '\n'
         return diff
 
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         repo_init = self._factory.repo_libgit2(wire)
 
         with repo_init as repo:
             swap = True
             flags = 0
             flags |= pygit2.GIT_DIFF_SHOW_BINARY
 
             if opt_ignorews:
                 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
 
             if commit_id_1 == self.EMPTY_COMMIT:
                 comm1 = repo[commit_id_2]
                 diff_obj = comm1.tree.diff_to_tree(
                     flags=flags, context_lines=context, swap=swap)
 
             else:
                 comm1 = repo[commit_id_2]
                 comm2 = repo[commit_id_1]
                 diff_obj = comm1.tree.diff_to_tree(
                     comm2.tree, flags=flags, context_lines=context, swap=swap)
             similar_flags = 0
             similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
             diff_obj.find_similar(flags=similar_flags)
 
             if file_filter:
                 for p in diff_obj:
                     if p.delta.old_file.path == file_filter:
-                        return BinaryEnvelope(p.data) or BinaryEnvelope(b'')
+                        return BytesEnvelope(p.data) or BytesEnvelope(b'')
                 # fo matching path == no diff
-                return BinaryEnvelope(b'')
-            return BinaryEnvelope(diff_obj.patch) or BinaryEnvelope(b'')
+                return BytesEnvelope(b'')
+
+            return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
 
     @reraise_safe_exceptions
     def node_history(self, wire, commit_id, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
             # optimize for n==1, rev-list is much faster for that use-case
             if limit == 1:
                 cmd = ['rev-list', '-1', commit_id, '--', path]
             else:
                 cmd = ['log']
                 if limit:
                     cmd.extend(['-n', str(safe_int(limit, 0))])
                 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
 
             output, __ = self.run_git_command(wire, cmd)
             commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
 
             return [x for x in commit_ids]
         return _node_history(context_uid, repo_id, commit_id, path, limit)
 
     @reraise_safe_exceptions
     def node_annotate_legacy(self, wire, commit_id, path):
         # note: replaced by pygit2 implementation
         cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for boundaries
         # -r commit_id ==> blames for the given commit
         output, __ = self.run_git_command(wire, cmd)
 
         result = []
         for i, blame_line in enumerate(output.splitlines()[:-1]):
             line_no = i + 1
             blame_commit_id, line = re.split(rb' ', blame_line, 1)
             result.append((line_no, blame_commit_id, line))
 
         return result
 
     @reraise_safe_exceptions
     def node_annotate(self, wire, commit_id, path):
 
         result_libgit = []
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             commit = repo[commit_id]
             blame_obj = repo.blame(path, newest_commit=commit_id)
             for i, line in enumerate(commit.tree[path].data.splitlines()):
                 line_no = i + 1
                 hunk = blame_obj.for_line(line_no)
                 blame_commit_id = hunk.final_commit_id.hex
 
                 result_libgit.append((line_no, blame_commit_id, line))
 
         return result_libgit
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id):
 
             cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
             try:
                 output, __ = self.run_git_command(wire, cmd)
                 return output.splitlines()
             except Exception:
                 # Can be raised for empty repositories
                 return []
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
             repo_init = self._factory.repo_libgit2(wire)
             from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
             results = []
             with repo_init as repo:
                 for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
                     results.append(commit.id.hex)
 
         return _get_all_commit_ids(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false',]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         if '_copts' in opts:
             _copts.extend(opts['_copts'] or [])
             del opts['_copts']
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
         _opts = {'env': gitenv, 'shell': False}
 
         proc = None
         try:
             _opts.update(opts)
             proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return b''.join(proc), b''.join(proc.stderr)
         except OSError as err:
             cmd = ' '.join(map(safe_str, cmd))  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
         finally:
             if proc:
                 proc.close()
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_git_hooks
         bare = self.bare(wire)
         path = wire['path']
         binary_dir = settings.BINARY_DIR
         if binary_dir:
             os.path.join(binary_dir, 'python3')
         return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_git_pre_hook_version, get_git_post_hook_version)
         bare = self.bare(wire)
         path = wire['path']
         return {
             'pre_version': get_git_pre_hook_version(path, bare),
             'post_version': get_git_post_hook_version(path, bare),
         }
 
     @reraise_safe_exceptions
     def set_head_ref(self, wire, head_name):
         log.debug('Setting refs/head to `%s`', head_name)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.set_head(f'refs/heads/{head_name}')
 
         return [head_name] + [f'set HEAD to refs/heads/{head_name}']
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def file_walker(_commit_id, path):
             repo_init = self._factory.repo_libgit2(wire)
 
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if path in ['', '/']:
                     tree = commit.tree
                 else:
                     tree = commit.tree[path.rstrip('/')]
                     tree_id = tree.id.hex
                     try:
                         tree = repo[tree_id]
                     except KeyError:
                         raise ObjectMissing(f'No tree with id: {tree_id}')
 
                 index = LibGit2Index.Index()
                 index.read_tree(tree)
                 file_iter = index
 
                 for file_node in file_iter:
                     file_path = file_node.path
                     mode = file_node.mode
                     is_link = stat.S_ISLNK(mode)
                     if mode == pygit2.GIT_FILEMODE_COMMIT:
                         log.debug('Skipping path %s as a commit node', file_path)
                         continue
                     yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)

vcsserver/remote/hg.py

@@ -1,1105 +1,1159 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 import binascii
 import io
 import logging
 import stat
 import urllib.request
 import urllib.parse
 import traceback
 import hashlib
 
 from hgext import largefiles, rebase, purge
 
 from mercurial import commands
 from mercurial import unionrepo
 from mercurial import verify
 from mercurial import repair
 
 import vcsserver
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, archive_repo, ArchiveNode, BinaryEnvelope
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, store_archive_in_cache, ArchiveNode, BytesEnvelope, \
+    BinaryEnvelope
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError,
     alwaysmatcher, patternmatcher, hgutil, hgext_strip)
 from vcsserver.str_utils import ascii_bytes, ascii_str, safe_str, safe_bytes
 from vcsserver.vcs_base import RemoteBase
+from vcsserver.config import hooks as hooks_config
+
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
 
     class LoggingUI(ui.ui):
 
         def status(self, *msg, **opts):
             str_msg = map(safe_str, msg)
             log.info(' '.join(str_msg).rstrip('\n'))
             #super(LoggingUI, self).status(*msg, **opts)
 
         def warn(self, *msg, **opts):
             str_msg = map(safe_str, msg)
             log.warning('ui_logger:'+' '.join(str_msg).rstrip('\n'))
             #super(LoggingUI, self).warn(*msg, **opts)
 
         def error(self, *msg, **opts):
             str_msg = map(safe_str, msg)
             log.error('ui_logger:'+' '.join(str_msg).rstrip('\n'))
             #super(LoggingUI, self).error(*msg, **opts)
 
         def note(self, *msg, **opts):
             str_msg = map(safe_str, msg)
             log.info('ui_logger:'+' '.join(str_msg).rstrip('\n'))
             #super(LoggingUI, self).note(*msg, **opts)
 
         def debug(self, *msg, **opts):
             str_msg = map(safe_str, msg)
             log.debug('ui_logger:'+' '.join(str_msg).rstrip('\n'))
             #super(LoggingUI, self).debug(*msg, **opts)
 
     baseui = LoggingUI()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(ascii_bytes(section), ascii_bytes(option), ascii_bytes(value))
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig(b'ui', b'quiet', b'true')
 
     baseui.setconfig(b'ui', b'paginate', b'never')
     # for better Error reporting of Mercurial
     baseui.setconfig(b'ui', b'message-output', b'stderr')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig(b'worker', b'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig(b'extensions', b'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig(b'extensions', b'largefiles', b'!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
 
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired) as e:
             raise_from_original(exceptions.AbortException(e), e)
         except RepoLookupError as e:
             raise_from_original(exceptions.LookupException(e), e)
         except RequirementError as e:
             raise_from_original(exceptions.RequirementException(e), e)
         except RepoError as e:
             raise_from_original(exceptions.VcsException(e), e)
         except LookupError as e:
             raise_from_original(exceptions.LookupException(e), e)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException(e), e)
 
             raise
     return wrapper
 
 
 class MercurialFactory(RepoFactory):
     repo_type = 'hg'
 
     def _create_config(self, config, hooks=True):
         if not hooks:
-            hooks_to_clean = frozenset((
-                'changegroup.repo_size', 'preoutgoing.pre_pull',
-                'outgoing.pull_logger', 'prechangegroup.pre_push'))
+
+            hooks_to_clean = {
+
+                hooks_config.HOOK_REPO_SIZE,
+                hooks_config.HOOK_PRE_PULL,
+                hooks_config.HOOK_PULL,
+
+                hooks_config.HOOK_PRE_PUSH,
+                # TODO: what about PRETXT, this was disabled in pre 5.0.0
+                hooks_config.HOOK_PRETX_PUSH,
+
+            }
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         repo = instance(baseui, safe_bytes(wire["path"]), create)
         log.debug('repository created: got HG object: %s', repo)
         return repo
 
     def repo(self, wire, create=False):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create)
 
 
 def patch_ui_message_output(baseui):
     baseui.setconfig(b'ui', b'quiet', b'false')
     output = io.BytesIO()
 
     def write(data, **unused_kwargs):
         output.write(data)
 
     baseui.status = write
     baseui.write = write
     baseui.warn = write
     baseui.debug = write
 
     return baseui, output
 
 
+def get_obfuscated_url(url_obj):
+    url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
+    url_obj.query = obfuscate_qs(url_obj.query)
+    obfuscated_uri = str(url_obj)
+    return obfuscated_uri
+
+
+def normalize_url_for_hg(url: str):
+    _proto = None
+
+    if '+' in url[:url.find('://')]:
+        _proto = url[0:url.find('+')]
+        url = url[url.find('+') + 1:]
+    return url, _proto
+
+
 class HgRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "obsolete": self.ctx_obsolete,
             "phase": self.ctx_phase,
             "hidden": self.ctx_hidden,
             "_file_paths": self.ctx_list,
         }
+        self._bulk_file_methods = {
+            "size": self.fctx_size,
+            "data": self.fctx_node_data,
+            "flags": self.fctx_flags,
+            "is_binary": self.is_binary,
+            "md5": self.md5_hash,
+        }
 
     def _get_ctx(self, repo, ref):
         return get_ctx(repo, ref)
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return safe_str(util.version())
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo = self._factory.repo(wire)
 
         try:
             return len(repo) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bookmarks(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo._bookmarks.items()}
 
         return _bookmarks(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _branches(_context_uid, _repo_id, _normal, _closed):
             repo = self._factory.repo(wire)
             iter_branches = repo.branchmap().iterbranches()
             bt = {}
             for branch_name, _heads, tip_node, is_closed in iter_branches:
                 if normal and not is_closed:
                     bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
                 if closed and is_closed:
                     bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
 
             return bt
 
         return _branches(context_uid, repo_id, normal, closed)
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, commit_id, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _commit_id, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
                     wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
                     result[attr] = method(wire, commit_id)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(
                         'Unknown bulk attribute: "%s"' % attr)
             return result
 
         return _bulk_request(repo_id, commit_id, sorted(pre_load))
 
     @reraise_safe_exceptions
     def ctx_branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_branch(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.branch()
         return _ctx_branch(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_date(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.date()
         return _ctx_date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.description()
 
     @reraise_safe_exceptions
     def ctx_files(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_files(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.files()
 
         return _ctx_files(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = self._get_ctx(repo, revision)
         return list(ctx)
 
     @reraise_safe_exceptions
     def ctx_parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_parents(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return [parent.hex() for parent in ctx.parents()
                     if not (parent.hidden() or parent.obsolete())]
 
         return _ctx_parents(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_children(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return [child.hex() for child in ctx.children()
                     if not (child.hidden() or child.obsolete())]
 
         return _ctx_children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_phase(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_phase(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             # public=0, draft=1, secret=3
             return ctx.phase()
         return _ctx_phase(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_obsolete(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.obsolete()
         return _ctx_obsolete(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_hidden(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_hidden(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.hidden()
         return _ctx_hidden(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
-        _proto = None
-        if '+' in url[:url.find('://')]:
-            _proto = url[0:url.find('+')]
-            url = url[url.find('+') + 1:]
+        url, _proto = normalize_url_for_hg(url)
+        url_obj = url_parser(safe_bytes(url))
+
+        test_uri = safe_str(url_obj.authinfo()[0])
+        authinfo = url_obj.authinfo()[1]
+        obfuscated_uri = get_obfuscated_url(url_obj)
+        log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
+
         handlers = []
-        url_obj = url_parser(url)
-        test_uri, authinfo = url_obj.authinfo()
-        url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
-        url_obj.query = obfuscate_qs(url_obj.query)
-
-        cleaned_uri = str(url_obj)
-        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
-
         if authinfo:
             # create a password manager
             passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "{}-{}".format('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.parse.urlencode(q)
         cu = "{}{}".format(test_uri, qs)
         req = urllib.request.Request(cu, None, {})
 
         try:
-            log.debug("Trying to open URL %s", cleaned_uri)
+            log.debug("Trying to open URL %s", obfuscated_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
-            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
+            log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
             # means it cannot be cloned
-            raise exceptions.URLError(e)("[{}] org_exc: {}".format(cleaned_uri, e))
+            raise exceptions.URLError(e)("[{}] org_exc: {}".format(obfuscated_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
-                    "Verifying if URL is a Mercurial repository: %s",
-                    cleaned_uri)
+                    "Verifying if URL is a Mercurial repository: %s", obfuscated_uri)
                 ui = make_ui_from_config(config)
-                peer_checker = makepeer(ui, url)
-                peer_checker.lookup('tip')
+                peer_checker = makepeer(ui, safe_bytes(url))
+                peer_checker.lookup(b'tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
-                        cleaned_uri)
+                        obfuscated_uri)
             raise exceptions.URLError(e)(
                 "url [%s] does not look like an hg repo org_exc: %s"
-                % (cleaned_uri, e))
+                % (obfuscated_uri, e))
 
-        log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
+        log.info("URL is a valid Mercurial repository: %s", obfuscated_uri)
         return True
 
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             # unpack the file-filter
             repo_path, node_path = file_filter
             match_filter = match(safe_bytes(repo_path), b'', [safe_bytes(node_path)])
         else:
             match_filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
 
         try:
             diff_iter = patch.diff(
                 repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts)
-            return BinaryEnvelope(b"".join(diff_iter))
+            return BytesEnvelope(b"".join(diff_iter))
         except RepoLookupError as e:
             raise exceptions.LookupException(e)()
 
     @reraise_safe_exceptions
     def node_history(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
             repo = self._factory.repo(wire)
 
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(safe_bytes(path))
 
             def history_iter():
                 limit_rev = fctx.rev()
                 for obj in reversed(list(fctx.filelog())):
                     obj = fctx.filectx(obj)
                     ctx = obj.changectx()
                     if ctx.hidden() or ctx.obsolete():
                         continue
 
                     if limit_rev >= obj.rev():
                         yield obj
 
             history = []
             for cnt, obj in enumerate(history_iter()):
                 if limit and cnt >= limit:
                     break
                 history.append(hex(obj.node()))
 
             return [x for x in history]
         return _node_history(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
     def node_history_untill(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history_until(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(safe_bytes(path))
 
             file_log = list(fctx.filelog())
             if limit:
                 # Limit to the last n items
                 file_log = file_log[-limit:]
 
             return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
         return _node_history_until(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
+
+    @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(safe_bytes(path))
 
         result = []
         for i, annotate_obj in enumerate(fctx.annotate(), 1):
             ln_no = i
             sha = hex(annotate_obj.fctx.node())
             content = annotate_obj.text
             result.append((ln_no, sha, content))
         return result
 
     @reraise_safe_exceptions
     def fctx_node_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(safe_bytes(path))
-        return BinaryEnvelope(fctx.data())
+        return BytesEnvelope(fctx.data())
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_flags(_repo_id, _commit_id, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             fctx = ctx.filectx(safe_bytes(path))
             return fctx.flags()
 
         return _fctx_flags(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def fctx_size(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_size(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             fctx = ctx.filectx(safe_bytes(path))
             return fctx.size()
         return _fctx_size(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id, _name):
             repo = self._factory.repo(wire)
             revs = [ascii_str(repo[x].hex()) for x in repo.filtered(b'visible').changelog.revs()]
             return revs
         return _get_all_commit_ids(context_uid, repo_id, name)
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(ascii_bytes(section), ascii_bytes(name), untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
             return largefiles.lfutil.isstandin(safe_bytes(path))
 
         return _is_large_file(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, revision, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _sha, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(safe_bytes(path))
             return fctx.isbinary()
 
         return _is_binary(repo_id, revision, path)
 
     @reraise_safe_exceptions
     def md5_hash(self, wire, revision, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _md5_hash(_repo_id, _sha, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(safe_bytes(path))
             return hashlib.md5(fctx.data()).hexdigest()
 
         return _md5_hash(repo_id, revision, path)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _lookup(_context_uid, _repo_id, _revision, _both):
-
             repo = self._factory.repo(wire)
             rev = _revision
             if isinstance(rev, int):
                 # NOTE(marcink):
                 # since Mercurial doesn't support negative indexes properly
                 # we need to shift accordingly by one to get proper index, e.g
                 # repo[-1] => repo[-2]
                 # repo[0]  => repo[-1]
                 if rev <= 0:
                     rev = rev + -1
             try:
                 ctx = self._get_ctx(repo, rev)
             except (TypeError, RepoLookupError, binascii.Error) as e:
                 e._org_exc_tb = traceback.format_exc()
                 raise exceptions.LookupException(e)(rev)
             except LookupError as e:
                 e._org_exc_tb = traceback.format_exc()
                 raise exceptions.LookupException(e)(e.name)
 
             if not both:
                 return ctx.hex()
 
             ctx = repo[ctx.hex()]
             return ctx.hex(), ctx.rev()
 
         return _lookup(context_uid, repo_id, revision, both)
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url):
         if not self.check_url(url, wire['config']):
             return
 
         repo = self._factory.repo(wire)
 
         # Disable any prompts for this repo
         repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         bookmarks = list(dict(repo._bookmarks).keys())
         remote = peer(repo, {}, safe_bytes(url))
         # Disable any prompts for this remote
         remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         return exchange.push(
             repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, rev)
         return ctx.rev()
 
     @reraise_safe_exceptions
     def rev_range(self, wire, commit_filter):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _rev_range(_context_uid, _repo_id, _filter):
             repo = self._factory.repo(wire)
             revisions = [
                 ascii_str(repo[rev].hex())
                 for rev in revrange(repo, list(map(ascii_bytes, commit_filter)))
             ]
             return revisions
 
         return _rev_range(context_uid, repo_id, sorted(commit_filter))
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [ascii_str(repo[r].hex()) for r in range(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         org_path = safe_bytes(wire["path"])
         other_path = safe_bytes(kwargs.pop('other_path', ''))
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.makeunionrepository(baseui, other_path, org_path)
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
 
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
         verify.verify(repo)
         return output.getvalue()
 
     @reraise_safe_exceptions
     def hg_update_cache(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
         with repo.wlock(), repo.lock():
             repo.updatecaches(full=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def hg_rebuild_fn_cache(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
 
         repair.rebuildfncache(baseui, repo)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def tags(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tags(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo.tags().items()}
 
         return _tags(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def update(self, wire, node='', clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         node = safe_bytes(node)
 
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [safe_bytes(branch)]
         else:
             args = []
         commands.heads(baseui, repo, template=b'{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
         a = changelog.ancestor(lookup(safe_bytes(revision1)), lookup(safe_bytes(revision2)))
         return hex(a)
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
         clone(baseui, safe_bytes(source), safe_bytes(dest), noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
 
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         publishing = baseui.configbool(b'phases', b'publish')
 
         def _filectxfn(_repo, ctx, path: bytes):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if safe_str(path) in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
                 if safe_bytes(node['path']) == path:
                     return memfilectx(
                         _repo,
                         changectx=ctx,
                         path=safe_bytes(node['path']),
                         data=safe_bytes(node['content']),
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copysource=False)
             abort_exc = exceptions.AbortException()
             raise abort_exc(f"Given path haven't been marked as added, changed or removed ({path})")
 
         if publishing:
             new_commit_phase = b'public'
         else:
             new_commit_phase = b'draft'
         with repo.ui.configoverride({(b'phases', b'new-commit'): new_commit_phase}):
             kwargs = {safe_bytes(k): safe_bytes(v) for k, v in extra.items()}
             commit_ctx = memctx(
                 repo=repo,
                 parents=parents,
                 text=safe_bytes(message),
                 files=[safe_bytes(x) for x in files],
                 filectxfn=_filectxfn,
                 user=safe_bytes(user),
                 date=(commit_time, commit_timezone),
                 extra=kwargs)
 
             n = repo.commitctx(commit_ctx)
             new_id = hex(n)
 
             return new_id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         # Disable any prompts for this repo
         repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         remote = peer(repo, {}, safe_bytes(url))
         # Disable any prompts for this remote
         remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
 
         return exchange.pull(
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
     def pull_cmd(self, wire, source, bookmark='', branch='', revision='', hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
         source = safe_bytes(source)
 
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
+
         if bookmark:
-            if isinstance(branch, list):
-                bookmark = [safe_bytes(x) for x in bookmark]
-            else:
-                bookmark = safe_bytes(bookmark)
-            opts['bookmark'] = bookmark
+            opts['bookmark'] = [safe_bytes(x) for x in bookmark] \
+                if isinstance(bookmark, list) else safe_bytes(bookmark)
+
         if branch:
-            if isinstance(branch, list):
-                branch = [safe_bytes(x) for x in branch]
-            else:
-                branch = safe_bytes(branch)
-            opts['branch'] = branch
+            opts['branch'] = [safe_bytes(x) for x in branch] \
+                if isinstance(branch, list) else safe_bytes(branch)
+
         if revision:
-            opts['rev'] = safe_bytes(revision)
+            opts['rev'] = [safe_bytes(x) for x in revision] \
+                if isinstance(revision, list) else safe_bytes(revision)
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
-    def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
+    def push(self, wire, revisions, dest_path, hooks: bool = True, push_branches: bool = False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
-        commands.push(baseui, repo, dest=dest_path, rev=revisions,
+
+        revisions = [safe_bytes(x) for x in revisions] \
+            if isinstance(revisions, list) else safe_bytes(revisions)
+
+        commands.push(baseui, repo, safe_bytes(dest_path),
+                      rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
-        hgext_strip(
+        hgext_strip.strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
     def get_unresolved_files(self, wire):
         repo = self._factory.repo(wire)
 
         log.debug('Calculating unresolved files for repo: %s', repo)
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui = self._factory._create_config(wire['config'])
         baseui.write = write
 
         commands.resolve(baseui, repo, list=True)
         unresolved = output.getvalue().splitlines(0)
         return unresolved
 
     @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig(b'ui', b'interactive', False)
-        commands.merge(baseui, repo, rev=revision)
+        commands.merge(baseui, repo, rev=safe_bytes(revision))
 
     @reraise_safe_exceptions
     def merge_state(self, wire):
         repo = self._factory.repo(wire)
         repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig(b'ui', b'interactive', False)
         ms = hg_merge.mergestate(repo)
         return [x for x in ms.unresolved()]
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username, close_branch=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        repo.ui.setconfig(b'ui', b'username', username)
-        commands.commit(baseui, repo, message=message, close_branch=close_branch)
+        repo.ui.setconfig(b'ui', b'username', safe_bytes(username))
+        commands.commit(baseui, repo, message=safe_bytes(message), close_branch=close_branch)
 
     @reraise_safe_exceptions
-    def rebase(self, wire, source=None, dest=None, abort=False):
+    def rebase(self, wire, source='', dest='', abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig(b'ui', b'interactive', False)
-        rebase.rebase(baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
+
+        rebase.rebase(baseui, repo, base=safe_bytes(source or ''), dest=safe_bytes(dest or ''),
+                      abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         node = ctx.node()
 
         date = (tag_time, tag_timezone)
         try:
-            hg_tag.tag(repo, name, node, message, local, user, date)
+            hg_tag.tag(repo, safe_bytes(name), node, safe_bytes(message), local, safe_bytes(user), date)
         except Abort as e:
             log.exception("Tag operation aborted")
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=''):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
+        revision = revision or ''
         commands.bookmark(baseui, repo, safe_bytes(bookmark), rev=safe_bytes(revision), force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         # we don't need any special hooks for Mercurial
         pass
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         return {
             'pre_version': vcsserver.__version__,
             'post_version': vcsserver.__version__,
         }
 
     @reraise_safe_exceptions
     def set_head_ref(self, wire, head_name):
         pass
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def file_walker(_commit_id, path):
             repo = self._factory.repo(wire)
             ctx = repo[_commit_id]
             is_root = path in ['', '/']
             if is_root:
                 matcher = alwaysmatcher(badfn=None)
             else:
                 matcher = patternmatcher('', [(b'glob', path+'/**', b'')], badfn=None)
             file_iter = ctx.manifest().walk(matcher)
 
             for fn in file_iter:
                 file_path = fn
                 flags = ctx.flags(fn)
                 mode = b'x' in flags and 0o755 or 0o644
                 is_link = b'l' in flags
 
                 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
 

vcsserver/remote/svn.py

@@ -1,890 +1,935 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 import os
 import subprocess
 from urllib.error import URLError
 import urllib.parse
 import logging
 import posixpath as vcspath
 import io
 import urllib.request
 import urllib.parse
 import urllib.error
 import traceback
 
 
 import svn.client  # noqa
 import svn.core  # noqa
 import svn.delta  # noqa
 import svn.diff  # noqa
 import svn.fs  # noqa
 import svn.repos  # noqa
 
 from vcsserver import svn_diff, exceptions, subprocessio, settings
-from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo, BinaryEnvelope
+from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
 from vcsserver.exceptions import NoContentException
 from vcsserver.str_utils import safe_str, safe_bytes
+from vcsserver.type_utils import assert_bytes
 from vcsserver.vcs_base import RemoteBase
 from vcsserver.lib.svnremoterepo import svnremoterepo
 log = logging.getLogger(__name__)
 
 
 svn_compatible_versions_map = {
     'pre-1.4-compatible': '1.3',
     'pre-1.5-compatible': '1.4',
     'pre-1.6-compatible': '1.5',
     'pre-1.8-compatible': '1.7',
     'pre-1.9-compatible': '1.8',
 }
 
 current_compatible_version = '1.14'
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting svn exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in svn remote call")
                 raise_from_original(exceptions.UnhandledException(e), e)
             raise
     return wrapper
 
 
 class SubversionFactory(RepoFactory):
     repo_type = 'svn'
 
     def _create_repo(self, wire, create, compatible_version):
         path = svn.core.svn_path_canonicalize(wire['path'])
         if create:
             fs_config = {'compatible-version': current_compatible_version}
             if compatible_version:
 
                 compatible_version_string = \
                     svn_compatible_versions_map.get(compatible_version) \
                     or compatible_version
                 fs_config['compatible-version'] = compatible_version_string
 
             log.debug('Create SVN repo with config `%s`', fs_config)
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
 
         log.debug('repository created: got SVN object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create, compatible_version)
 
 
 NODE_TYPE_MAPPING = {
     svn.core.svn_node_file: 'file',
     svn.core.svn_node_dir: 'dir',
 }
 
 
 class SvnRemote(RemoteBase):
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
 
+        self._bulk_methods = {
+            # NOT supported in SVN ATM...
+        }
+        self._bulk_file_methods = {
+            "size": self.get_file_size,
+            "data": self.get_file_content,
+            "flags": self.get_node_type,
+            "is_binary": self.is_binary,
+            "md5": self.md5_hash
+        }
+
+    @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        # since we use unified API, we need to cast from str to in for SVN
+        commit_id = int(commit_id)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
+
     @reraise_safe_exceptions
     def discover_svn_version(self):
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
         return safe_str(svn_ver)
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
-
         try:
             return self.lookup(wire, -1) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
-    def check_url(self, url):
+    def check_url(self, url, config):
 
-        # uuid function get's only valid UUID from proper repo, else
+        # uuid function gets only valid UUID from proper repo, else
         # throws exception
         username, password, src_url = self.get_url_and_credentials(url)
         try:
-            svnremoterepo(username, password, src_url).svn().uuid
+            svnremoterepo(safe_bytes(username), safe_bytes(password), safe_bytes(src_url)).svn().uuid
         except Exception:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
-            raise URLError(
-                '"{}" is not a valid Subversion source url.'.format(url))
+            raise URLError(f'"{url}" is not a valid Subversion source url.')
         return True
 
     def is_path_valid_repository(self, wire, path):
 
         # NOTE(marcink):  short circuit the check for SVN repo
         # the repos.open might be expensive to check, but we have one cheap
         # pre condition that we can use, to check for 'format' file
 
         if not os.path.isfile(os.path.join(path, 'format')):
             return False
 
         try:
             svn.repos.open(path)
         except svn.core.SubversionException:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
             return False
         return True
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         cmd = ['svnadmin', 'info', repo_path]
         stdout, stderr = subprocessio.run_command(cmd)
         return stdout
 
+    @reraise_safe_exceptions
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
+    @reraise_safe_exceptions
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         start_rev = None
         end_rev = None
         if start_ts:
             start_ts_svn = apr_time_t(start_ts)
             start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
         else:
             start_rev = 1
         if end_ts:
             end_ts_svn = apr_time_t(end_ts)
             end_rev = svn.repos.dated_revision(repo, end_ts_svn)
         else:
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
+    @reraise_safe_exceptions
     def revision_properties(self, wire, revision):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _revision_properties(_repo_id, _revision):
             repo = self._factory.repo(wire)
             fs_ptr = svn.repos.fs(repo)
             return svn.fs.revision_proplist(fs_ptr, revision)
         return _revision_properties(repo_id, revision)
 
     def revision_changes(self, wire, revision):
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         editor = svn.repos.ChangeCollector(fsobj, rev_root)
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         base_dir = ""
         send_deltas = False
         svn.repos.replay2(
             rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
             editor_ptr, editor_baton, None)
 
         added = []
         changed = []
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
         for path, change in editor.changes.items():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
             if change.item_kind == svn.core.svn_node_dir:
                 continue
             if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
                                    svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(
                     "Action {} not supported on path {}".format(
                         change.action, path))
 
         changes = {
             'added': added,
             'changed': changed,
             'removed': removed,
         }
         return changes
 
     @reraise_safe_exceptions
     def node_history(self, wire, path, revision, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
             cross_copies = False
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             rev_root = svn.fs.revision_root(fsobj, revision)
 
             history_revisions = []
             history = svn.fs.node_history(rev_root, path)
             history = svn.fs.history_prev(history, cross_copies)
             while history:
                 __, node_revision = svn.fs.history_location(history)
                 history_revisions.append(node_revision)
                 if limit and len(history_revisions) >= limit:
                     break
                 history = svn.fs.history_prev(history, cross_copies)
             return history_revisions
         return _assert_correct_path(context_uid, repo_id, path, revision, limit)
 
+    @reraise_safe_exceptions
     def node_properties(self, wire, path, revision):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_properties(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             rev_root = svn.fs.revision_root(fsobj, revision)
             return svn.fs.node_proplist(rev_root, path)
         return _node_properties(repo_id, path, revision)
 
     def file_annotate(self, wire, path, revision):
         abs_path = 'file://' + urllib.request.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
         start_rev = svn_opt_revision_value_t(0)
         peg_rev = svn_opt_revision_value_t(revision)
         end_rev = peg_rev
 
         annotations = []
 
         def receiver(line_no, revision, author, date, line, pool):
             annotations.append((line_no, revision, line))
 
         # TODO: Cannot use blame5, missing typemap function in the swig code
         try:
             svn.client.blame2(
                 file_uri, peg_rev, start_rev, end_rev,
                 receiver, svn.client.create_context())
         except svn.core.SubversionException as exc:
             log.exception("Error during blame operation.")
             raise Exception(
                 "Blame not supported or file does not exist at path %s. "
                 "Error %s." % (path, exc))
 
         return annotations
 
-    def get_node_type(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_node_type(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _get_node_type(_repo_id, _path, _revision):
+        def _get_node_type(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             fs_ptr = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_rev(fs_ptr)
             root = svn.fs.revision_root(fs_ptr, _revision)
             node = svn.fs.check_path(root, path)
             return NODE_TYPE_MAPPING.get(node, None)
-        return _get_node_type(repo_id, path, revision)
+        return _get_node_type(repo_id, revision, path)
 
-    def get_nodes(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_nodes(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_nodes(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_rev(fsobj)
             root = svn.fs.revision_root(fsobj, _revision)
             entries = svn.fs.dir_entries(root, path)
             result = []
             for entry_path, entry_info in entries.items():
                 result.append(
                     (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
             return result
         return _get_nodes(repo_id, path, revision)
 
-    def get_file_content(self, wire, path, rev=None):
+    @reraise_safe_exceptions
+    def get_file_content(self, wire, rev=None, path=''):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
+
         if rev is None:
-            rev = svn.fs.youngest_revision(fsobj)
+            rev = svn.fs.youngest_rev(fsobj)
+
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
-        return BinaryEnvelope(content.read())
+        return BytesEnvelope(content.read())
 
-    def get_file_size(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_file_size(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _get_file_size(_repo_id, _path, _revision):
+        def _get_file_size(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_revision(fsobj)
             root = svn.fs.revision_root(fsobj, _revision)
             size = svn.fs.file_length(root, path)
             return size
-        return _get_file_size(repo_id, path, revision)
+        return _get_file_size(repo_id, revision, path)
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
-    def get_url_and_credentials(self, src_url):
+    def get_url_and_credentials(self, src_url) -> tuple[str, str, str]:
         obj = urllib.parse.urlparse(src_url)
-        username = obj.username or None
-        password = obj.password or None
+        username = obj.username or ''
+        password = obj.password or ''
         return username, password, src_url
 
     def import_remote_repository(self, wire, src_url):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         username, password, src_url = self.get_url_and_credentials(src_url)
         rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
                      '--trust-server-cert-failures=unknown-ca']
         if username and password:
             rdump_cmd += ['--username', username, '--password', password]
         rdump_cmd += [src_url]
 
         rdump = subprocess.Popen(
             rdump_cmd,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         load = subprocess.Popen(
             ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
 
         # TODO: johbo: This can be a very long operation, might be better
         # to track some kind of status and provide an api to check if the
         # import is done.
         rdump.wait()
         load.wait()
 
         log.debug('Return process ended with code: %s', rdump.returncode)
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
             log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
 
             reason = 'UNKNOWN'
             if b'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
 
             if reason == 'UNKNOWN':
                 reason = f'UNKNOWN:{safe_str(errors)}'
 
             raise Exception(
                 'Failed to dump the remote repository from {}. Reason:{}'.format(
                     src_url, reason))
         if load.returncode != 0:
             raise Exception(
                 'Failed to load the dump of remote repository from %s.' %
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
 
-        updated = [{k: safe_bytes(v) for k, v in x.items() if isinstance(v, str)} for x in updated]
-
         message = safe_bytes(message)
         author = safe_bytes(author)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
 
         rev = svn.fs.youngest_rev(fsobj)
         txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
         txn_root = svn.fs.txn_root(txn)
 
         for node in updated:
             TxnNodeProcessor(node, txn_root).update()
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp:
-            apr_time = int(apr_time_t(timestamp))
+            apr_time = apr_time_t(timestamp)
             ts_formatted = svn.core.svn_time_to_cstring(apr_time)
             svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
 
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
+    @reraise_safe_exceptions
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
         wire.update(cache=False)
         repo = self._factory.repo(wire)
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
-            return BinaryEnvelope(diff_creator.generate_diff())
+            return BytesEnvelope(diff_creator.generate_diff())
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
                 "Path might not exist %s, %s", path1, path2)
-            return BinaryEnvelope(b'')
+            return BytesEnvelope(b'')
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return False
 
     @reraise_safe_exceptions
     def is_binary(self, wire, rev, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _rev, _path):
-            raw_bytes = self.get_file_content(wire, path, rev)
-            return raw_bytes and b'\0' in raw_bytes
+            raw_bytes = self.get_file_content(wire, rev, path)
+            if not raw_bytes:
+                return False
+            return b'\0' in raw_bytes
 
         return _is_binary(repo_id, rev, path)
 
     @reraise_safe_exceptions
     def md5_hash(self, wire, rev, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
 
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _md5_hash(_repo_id, _rev, _path):
             return ''
 
         return _md5_hash(repo_id, rev, path)
 
     @reraise_safe_exceptions
     def run_svn_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         safe_call = opts.pop('_safe', False)
 
         svnenv = os.environ.copy()
         svnenv.update(opts.pop('extra_env', {}))
 
         _opts = {'env': svnenv, 'shell': False}
 
         try:
             _opts.update(opts)
             proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return b''.join(proc), b''.join(proc.stderr)
         except OSError as err:
             if safe_call:
                 return '', safe_str(err).strip()
             else:
                 cmd = ' '.join(map(safe_str, cmd))  # human friendly CMD
                 tb_err = ("Couldn't run svn command (%s).\n"
                           "Original error was:%s\n"
                           "Call options:%s\n"
                           % (cmd, err, _opts))
                 log.exception(tb_err)
                 raise exceptions.VcsException()(tb_err)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_svn_hooks
         repo_path = wire['path']
         binary_dir = settings.BINARY_DIR
         executable = None
         if binary_dir:
             executable = os.path.join(binary_dir, 'python3')
         return install_svn_hooks(repo_path, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_svn_pre_hook_version, get_svn_post_hook_version)
         repo_path = wire['path']
         return {
             'pre_version': get_svn_pre_hook_version(repo_path),
             'post_version': get_svn_post_hook_version(repo_path),
         }
 
     @reraise_safe_exceptions
     def set_head_ref(self, wire, head_name):
         pass
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def walk_tree(root, root_dir, _commit_id):
             """
             Special recursive svn repo walker
             """
             root_dir = safe_bytes(root_dir)
 
             filemode_default = 0o100644
             filemode_executable = 0o100755
 
             file_iter = svn.fs.dir_entries(root, root_dir)
             for f_name in file_iter:
                 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
 
                 if f_type == 'dir':
                     # return only DIR, and then all entries in that dir
                     yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
                     new_root = os.path.join(root_dir, f_name)
                     yield from walk_tree(root, new_root, _commit_id)
                 else:
 
                     f_path = os.path.join(root_dir, f_name).rstrip(b'/')
                     prop_list = svn.fs.node_proplist(root, f_path)
 
                     f_mode = filemode_default
                     if prop_list.get('svn:executable'):
                         f_mode = filemode_executable
 
                     f_is_link = False
                     if prop_list.get('svn:special'):
                         f_is_link = True
 
                     data = {
                         'is_link': f_is_link,
                         'mode': f_mode,
                         'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
                     }
 
                     yield f_path, data, f_type
 
         def file_walker(_commit_id, path):
             repo = self._factory.repo(wire)
             root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
 
             def no_content():
                 raise NoContentException()
 
             for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
                 file_path = f_name
 
                 if f_type == 'dir':
                     mode = f_data['mode']
                     yield ArchiveNode(file_path, mode, False, no_content)
                 else:
                     mode = f_data['mode']
                     is_link = f_data['is_link']
                     data_stream = f_data['content_stream']
                     yield ArchiveNode(file_path, mode, is_link, data_stream)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
 
 
 class SvnDiffer(object):
     """
     Utility to create diffs based on difflib and the Subversion api
     """
 
     binary_content = False
 
     def __init__(
             self, repo, src_rev, src_path, tgt_rev, tgt_path,
             ignore_whitespace, context):
         self.repo = repo
         self.ignore_whitespace = ignore_whitespace
         self.context = context
 
         fsobj = svn.repos.fs(repo)
 
         self.tgt_rev = tgt_rev
         self.tgt_path = tgt_path or ''
         self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
         self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
 
         self.src_rev = src_rev
         self.src_path = src_path or self.tgt_path
         self.src_root = svn.fs.revision_root(fsobj, src_rev)
         self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
 
         self._validate()
 
     def _validate(self):
         if (self.tgt_kind != svn.core.svn_node_none and
                 self.src_kind != svn.core.svn_node_none and
                 self.src_kind != self.tgt_kind):
             # TODO: johbo: proper error handling
             raise Exception(
                 "Source and target are not compatible for diff generation. "
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
-    def generate_diff(self):
-        buf = io.StringIO()
+    def generate_diff(self) -> bytes:
+        buf = io.BytesIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
-    def _generate_dir_diff(self, buf):
+    def _generate_dir_diff(self, buf: io.BytesIO):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
             self.src_root,
             self.src_path,
             '',  # src_entry
             self.tgt_root,
             self.tgt_path,
             editor_ptr, editor_baton,
             authorization_callback_allow_all,
             False,  # text_deltas
             svn.core.svn_depth_infinity,  # depth
             False,  # entry_props
             False,  # ignore_ancestry
         )
 
         for path, __, change in sorted(editor.changes):
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
-    def _generate_file_diff(self, buf):
+    def _generate_file_diff(self, buf: io.BytesIO):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
         elif self.tgt_kind == svn.core.svn_node_none:
             change = "delete"
         tgt_base, tgt_path = vcspath.split(self.tgt_path)
         src_base, src_path = vcspath.split(self.src_path)
         self._generate_node_diff(
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
-            self, buf, change, tgt_path, tgt_base, src_path, src_base):
-
+            self, buf: io.BytesIO, change, tgt_path, tgt_base, src_path, src_base):
 
+        tgt_path_bytes = safe_bytes(tgt_path)
         tgt_path = safe_str(tgt_path)
+
+        src_path_bytes = safe_bytes(src_path)
         src_path = safe_str(src_path)
 
-
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
             # we compare same revisions
             return
 
         tgt_full_path = vcspath.join(tgt_base, tgt_path)
         src_full_path = vcspath.join(src_base, src_path)
 
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
-        if mime_type and not mime_type.startswith('text'):
+        if mime_type and not mime_type.startswith(b'text'):
             self.binary_content = True
-            buf.write("=" * 67 + '\n')
-            buf.write("Cannot display: file marked as a binary type.\n")
-            buf.write("svn:mime-type = %s\n" % mime_type)
-        buf.write("Index: {}\n".format(tgt_path))
-        buf.write("=" * 67 + '\n')
-        buf.write("diff --git a/{tgt_path} b/{tgt_path}\n".format(
-            tgt_path=tgt_path))
+            buf.write(b"=" * 67 + b'\n')
+            buf.write(b"Cannot display: file marked as a binary type.\n")
+            buf.write(b"svn:mime-type = %s\n" % mime_type)
+        buf.write(b"Index: %b\n" % tgt_path_bytes)
+        buf.write(b"=" * 67 + b'\n')
+        buf.write(b"diff --git a/%b b/%b\n" % (tgt_path_bytes, tgt_path_bytes))
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
-            buf.write("new file mode 10644\n")
+            buf.write(b"new file mode 10644\n")
+
+            # TODO(marcink): intro to binary detection of svn patches
+            # if self.binary_content:
+            #     buf.write(b'GIT binary patch\n')
+
+            buf.write(b"--- /dev/null\t(revision 0)\n")
+            src_lines = []
+        else:
+            if change == 'delete':
+                buf.write(b"deleted file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
-            buf.write("--- /dev/null\t(revision 0)\n")
-            src_lines = []
-        else:
-            if change == 'delete':
-                buf.write("deleted file mode 10644\n")
-
-            #TODO(marcink): intro to binary detection of svn patches
-            # if self.binary_content:
-            #     buf.write('GIT binary patch\n')
-
-            buf.write("--- a/{}\t(revision {})\n".format(
-                src_path, self.src_rev))
+            buf.write(b"--- a/%b\t(revision %d)\n" % (src_path_bytes, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
-            buf.write("+++ /dev/null\t(revision {})\n".format(self.tgt_rev))
+            buf.write(b"+++ /dev/null\t(revision %d)\n" % self.tgt_rev)
             tgt_lines = []
         else:
-            buf.write("+++ b/{}\t(revision {})\n".format(
-                tgt_path, self.tgt_rev))
+            buf.write(b"+++ b/%b\t(revision %d)\n" % (tgt_path_bytes, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
+        # we made our diff header, time to generate the diff content into our buffer
+
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
 
             buf.writelines(udiff)
 
-    def _get_mime_type(self, path):
+    def _get_mime_type(self, path) -> bytes:
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
         except svn.core.SubversionException:
             mime_type = svn.fs.node_prop(
                 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
         return mime_type
 
     def _svn_readlines(self, fs_root, node_path):
         if self.binary_content:
             return []
         node_kind = svn.fs.check_path(fs_root, node_path)
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
         content = svn.core.Stream(
             svn.fs.file_contents(fs_root, node_path)).read()
 
         return content.splitlines(True)
 
 
 class DiffChangeEditor(svn.delta.Editor):
     """
     Records changes between two given revisions
     """
 
     def __init__(self):
         self.changes = []
 
     def delete_entry(self, path, revision, parent_baton, pool=None):
         self.changes.append((path, None, 'delete'))
 
     def add_file(
             self, path, parent_baton, copyfrom_path, copyfrom_revision,
             file_pool=None):
         self.changes.append((path, 'file', 'add'))
 
     def open_file(self, path, parent_baton, base_revision, file_pool=None):
         self.changes.append((path, 'file', 'change'))
 
 
 def authorization_callback_allow_all(root, path, pool):
     return True
 
 
 class TxnNodeProcessor(object):
     """
     Utility to process the change of one node within a transaction root.
 
     It encapsulates the knowledge of how to add, update or remove
     a node for a given transaction root. The purpose is to support the method
     `SvnRemote.commit`.
     """
 
     def __init__(self, node, txn_root):
-        assert isinstance(node['path'], bytes)
+        assert_bytes(node['path'])
 
         self.node = node
         self.txn_root = txn_root
 
     def update(self):
         self._ensure_parent_dirs()
         self._add_file_if_node_does_not_exist()
         self._update_file_content()
         self._update_file_properties()
 
     def remove(self):
         svn.fs.delete(self.txn_root, self.node['path'])
         # TODO: Clean up directory if empty
 
     def _ensure_parent_dirs(self):
         curdir = vcspath.dirname(self.node['path'])
         dirs_to_create = []
         while not self._svn_path_exists(curdir):
             dirs_to_create.append(curdir)
             curdir = vcspath.dirname(curdir)
 
         for curdir in reversed(dirs_to_create):
             log.debug('Creating missing directory "%s"', curdir)
             svn.fs.make_dir(self.txn_root, curdir)
 
     def _svn_path_exists(self, path):
         path_status = svn.fs.check_path(self.txn_root, path)
         return path_status != svn.core.svn_node_none
 
     def _add_file_if_node_does_not_exist(self):
         kind = svn.fs.check_path(self.txn_root, self.node['path'])
         if kind == svn.core.svn_node_none:
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
-        assert isinstance(self.node['content'], bytes)
+        assert_bytes(self.node['content'])
 
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
         for key, value in properties.items():
             svn.fs.change_node_prop(
-                self.txn_root, self.node['path'], key, value)
+                self.txn_root, self.node['path'], safe_bytes(key), safe_bytes(value))
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
-    return timestamp * 1E6
+    return int(timestamp * 1E6)
 
 
 def svn_opt_revision_value_t(num):
     """
     Put `num` into a `svn_opt_revision_value_t` structure.
     """
     value = svn.core.svn_opt_revision_value_t()
     value.number = num
     revision = svn.core.svn_opt_revision_t()
     revision.kind = svn.core.svn_opt_revision_number
     revision.value = value
     return revision

vcsserver/svn_diff.py

@@ -1,209 +1,211 b''
 #
 # Copyright (C) 2004-2009 Edgewall Software
 # Copyright (C) 2004-2006 Christopher Lenz <[e-mail]>
 # All rights reserved.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
 # are also available at http://trac.edgewall.org/wiki/TracLicense.
 #
 # This software consists of voluntary contributions made by many
 # individuals. For the exact contribution history, see the revision
 # history and logs, available at http://trac.edgewall.org/log/.
 #
 # Author: Christopher Lenz <[e-mail]>
 
 import difflib
 
 
-def get_filtered_hunks(fromlines, tolines, context=None,
-                       ignore_blank_lines=False, ignore_case=False,
-                       ignore_space_changes=False):
+def get_filtered_hunks(from_lines, to_lines, context=None,
+                       ignore_blank_lines: bool = False, ignore_case: bool = False,
+                       ignore_space_changes: bool = False):
     """Retrieve differences in the form of `difflib.SequenceMatcher`
     opcodes, grouped according to the ``context`` and ``ignore_*``
     parameters.
 
-    :param fromlines: list of lines corresponding to the old content
-    :param tolines: list of lines corresponding to the new content
+    :param from_lines: list of lines corresponding to the old content
+    :param to_lines: list of lines corresponding to the new content
     :param ignore_blank_lines: differences about empty lines only are ignored
     :param ignore_case: upper case / lower case only differences are ignored
     :param ignore_space_changes: differences in amount of spaces are ignored
     :param context: the number of "equal" lines kept for representing
                     the context of the change
     :return: generator of grouped `difflib.SequenceMatcher` opcodes
 
     If none of the ``ignore_*`` parameters is `True`, there's nothing
     to filter out the results will come straight from the
     SequenceMatcher.
     """
-    hunks = get_hunks(fromlines, tolines, context)
+    hunks = get_hunks(from_lines, to_lines, context)
     if ignore_space_changes or ignore_case or ignore_blank_lines:
-        hunks = filter_ignorable_lines(hunks, fromlines, tolines, context,
+        hunks = filter_ignorable_lines(hunks, from_lines, to_lines, context,
                                        ignore_blank_lines, ignore_case,
                                        ignore_space_changes)
     return hunks
 
 
-def get_hunks(fromlines, tolines, context=None):
+def get_hunks(from_lines, to_lines, context=None):
     """Generator yielding grouped opcodes describing differences .
 
     See `get_filtered_hunks` for the parameter descriptions.
     """
-    matcher = difflib.SequenceMatcher(None, fromlines, tolines)
+    matcher = difflib.SequenceMatcher(None, from_lines, to_lines)
     if context is None:
         return (hunk for hunk in [matcher.get_opcodes()])
     else:
         return matcher.get_grouped_opcodes(context)
 
 
-def filter_ignorable_lines(hunks, fromlines, tolines, context,
+def filter_ignorable_lines(hunks, from_lines, to_lines, context,
                            ignore_blank_lines, ignore_case,
                            ignore_space_changes):
     """Detect line changes that should be ignored and emits them as
     tagged as "equal", possibly joined with the preceding and/or
     following "equal" block.
 
     See `get_filtered_hunks` for the parameter descriptions.
     """
     def is_ignorable(tag, fromlines, tolines):
+
         if tag == 'delete' and ignore_blank_lines:
-            if ''.join(fromlines) == '':
+            if b''.join(fromlines) == b'':
                 return True
         elif tag == 'insert' and ignore_blank_lines:
-            if ''.join(tolines) == '':
+            if b''.join(tolines) == b'':
                 return True
         elif tag == 'replace' and (ignore_case or ignore_space_changes):
             if len(fromlines) != len(tolines):
                 return False
 
             def f(input_str):
                 if ignore_case:
                     input_str = input_str.lower()
                 if ignore_space_changes:
-                    input_str = ' '.join(input_str.split())
+                    input_str = b' '.join(input_str.split())
                 return input_str
 
             for i in range(len(fromlines)):
                 if f(fromlines[i]) != f(tolines[i]):
                     return False
             return True
 
     hunks = list(hunks)
     opcodes = []
     ignored_lines = False
     prev = None
     for hunk in hunks:
         for tag, i1, i2, j1, j2 in hunk:
             if tag == 'equal':
                 if prev:
                     prev = (tag, prev[1], i2, prev[3], j2)
                 else:
                     prev = (tag, i1, i2, j1, j2)
             else:
-                if is_ignorable(tag, fromlines[i1:i2], tolines[j1:j2]):
+                if is_ignorable(tag, from_lines[i1:i2], to_lines[j1:j2]):
                     ignored_lines = True
                     if prev:
                         prev = 'equal', prev[1], i2, prev[3], j2
                     else:
                         prev = 'equal', i1, i2, j1, j2
                     continue
                 if prev:
                     opcodes.append(prev)
                 opcodes.append((tag, i1, i2, j1, j2))
                 prev = None
     if prev:
         opcodes.append(prev)
 
     if ignored_lines:
         if context is None:
             yield opcodes
         else:
             # we leave at most n lines with the tag 'equal' before and after
             # every change
             n = context
             nn = n + n
 
             group = []
+
             def all_equal():
                 all(op[0] == 'equal' for op in group)
             for idx, (tag, i1, i2, j1, j2) in enumerate(opcodes):
                 if idx == 0 and tag == 'equal': # Fixup leading unchanged block
                     i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
                 elif tag == 'equal' and i2 - i1 > nn:
                     group.append((tag, i1, min(i2, i1 + n), j1,
                                   min(j2, j1 + n)))
                     if not all_equal():
                         yield group
                     group = []
                     i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
                 group.append((tag, i1, i2, j1, j2))
 
             if group and not (len(group) == 1 and group[0][0] == 'equal'):
                 if group[-1][0] == 'equal': # Fixup trailing unchanged block
                     tag, i1, i2, j1, j2 = group[-1]
                     group[-1] = tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)
                 if not all_equal():
                     yield group
     else:
         for hunk in hunks:
             yield hunk
 
 
-NO_NEWLINE_AT_END = '\\ No newline at end of file'
+NO_NEWLINE_AT_END = b'\\ No newline at end of file'
+LINE_TERM = b'\n'
 
 
-def unified_diff(fromlines, tolines, context=None, ignore_blank_lines=0,
-                 ignore_case=0, ignore_space_changes=0, lineterm='\n'):
+def unified_diff(from_lines, to_lines, context=None, ignore_blank_lines: bool = False,
+                 ignore_case: bool = False, ignore_space_changes: bool = False, lineterm=LINE_TERM) -> bytes:
     """
     Generator producing lines corresponding to a textual diff.
 
     See `get_filtered_hunks` for the parameter descriptions.
     """
     # TODO: johbo: Check if this can be nicely integrated into the matching
 
     if ignore_space_changes:
-        fromlines = [l.strip() for l in fromlines]
-        tolines = [l.strip() for l in tolines]
+        from_lines = [l.strip() for l in from_lines]
+        to_lines = [l.strip() for l in to_lines]
 
-    for group in get_filtered_hunks(fromlines, tolines, context,
+    def _hunk_range(start, length) -> bytes:
+        if length != 1:
+            return b'%d,%d' % (start, length)
+        else:
+            return b'%d' % (start,)
+
+    for group in get_filtered_hunks(from_lines, to_lines, context,
                                     ignore_blank_lines, ignore_case,
                                     ignore_space_changes):
         i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
         if i1 == 0 and i2 == 0:
             i1, i2 = -1, -1  # support for Add changes
         if j1 == 0 and j2 == 0:
             j1, j2 = -1, -1  # support for Delete changes
-        yield '@@ -{} +{} @@{}'.format(
+        yield b'@@ -%b +%b @@%b' % (
             _hunk_range(i1 + 1, i2 - i1),
             _hunk_range(j1 + 1, j2 - j1),
             lineterm)
         for tag, i1, i2, j1, j2 in group:
             if tag == 'equal':
-                for line in fromlines[i1:i2]:
+                for line in from_lines[i1:i2]:
                     if not line.endswith(lineterm):
-                        yield ' ' + line + lineterm
-                        yield NO_NEWLINE_AT_END + lineterm
-                    else:
-                        yield ' ' + line
-            else:
-                if tag in ('replace', 'delete'):
-                    for line in fromlines[i1:i2]:
-                        if not line.endswith(lineterm):
-                            yield '-' + line + lineterm
+                        yield b' ' + line + lineterm
                             yield NO_NEWLINE_AT_END + lineterm
                         else:
-                            yield '-' + line
-                if tag in ('replace', 'insert'):
-                    for line in tolines[j1:j2]:
+                        yield b' ' + line
+            else:
+                if tag in ('replace', 'delete'):
+                    for line in from_lines[i1:i2]:
                         if not line.endswith(lineterm):
-                            yield '+' + line + lineterm
+                            yield b'-' + line + lineterm
                             yield NO_NEWLINE_AT_END + lineterm
                         else:
-                            yield '+' + line
-
-
-def _hunk_range(start, length):
-    if length != 1:
-        return '%d,%d' % (start, length)
+                            yield b'-' + line
+                if tag in ('replace', 'insert'):
+                    for line in to_lines[j1:j2]:
+                        if not line.endswith(lineterm):
+                            yield b'+' + line + lineterm
+                            yield NO_NEWLINE_AT_END + lineterm
     else:
-        return '%d' % (start, )
+                            yield b'+' + line

vcsserver/tests/test_svn.py

@@ -1,103 +1,103 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import mock
 import pytest
 import sys
 
 from vcsserver.str_utils import ascii_bytes
 
 
 class MockPopen(object):
     def __init__(self, stderr):
         self.stdout = io.BytesIO(b'')
         self.stderr = io.BytesIO(stderr)
         self.returncode = 1
 
     def wait(self):
         pass
 
 
 INVALID_CERTIFICATE_STDERR = '\n'.join([
     'svnrdump: E230001: Unable to connect to a repository at URL url',
     'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
 ])
 
 
 @pytest.mark.parametrize('stderr,expected_reason', [
     (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
     ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
 @pytest.mark.xfail(sys.platform == "cygwin",
                    reason="SVN not packaged for Cygwin")
 def test_import_remote_repository_certificate_error(stderr, expected_reason):
     from vcsserver.remote import svn
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
     remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     with mock.patch('subprocess.Popen',
                     return_value=MockPopen(ascii_bytes(stderr))):
         with pytest.raises(Exception) as excinfo:
             remote.import_remote_repository({'path': 'path'}, 'url')
 
     expected_error_args = 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason)
 
     assert excinfo.value.args[0] == expected_error_args
 
 
 def test_svn_libraries_can_be_imported():
     import svn.client
     assert svn.client is not None
 
 
 @pytest.mark.parametrize('example_url, parts', [
-    ('http://server.com', (None, None, 'http://server.com')),
-    ('http://[e-mail]', ('user', None, 'http://[e-mail]')),
+    ('http://server.com', ('', '', 'http://server.com')),
+    ('http://[e-mail]', ('user', '', 'http://[e-mail]')),
     ('http://user:[e-mail]', ('user', 'pass', 'http://user:[e-mail]')),
-    ('<script>', (None, None, '<script>')),
-    ('http://', (None, None, 'http://')),
+    ('<script>', ('', '', '<script>')),
+    ('http://', ('', '', 'http://')),
 ])
 def test_username_password_extraction_from_url(example_url, parts):
     from vcsserver.remote import svn
 
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
     remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     assert remote.get_url_and_credentials(example_url) == parts
 
 
 @pytest.mark.parametrize('call_url', [
     b'https://svn.code.sf.net/p/svnbook/source/trunk/',
     b'https://[e-mail]/p/svnbook/source/trunk/',
     b'https://marcink:[e-mail]/p/svnbook/source/trunk/',
 ])
 def test_check_url(call_url):
     from vcsserver.remote import svn
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
     remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
-    assert remote.check_url(call_url)
+    assert remote.check_url(call_url, {'dummy': 'config'})
 

vcsserver/tweens/request_wrapper.py

@@ -1,112 +1,123 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
-
+import base64
 import time
 import logging
 
+import msgpack
+
 import vcsserver
 from vcsserver.str_utils import safe_str, ascii_str
 
 log = logging.getLogger(__name__)
 
 
 def get_access_path(environ):
     path = environ.get('PATH_INFO')
     return path
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
-def get_call_context(registry) -> dict:
+def get_call_context(request) -> dict:
     cc = {}
+    registry = request.registry
     if hasattr(registry, 'vcs_call_context'):
         cc.update({
             'X-RC-Method': registry.vcs_call_context.get('method'),
             'X-RC-Repo-Name': registry.vcs_call_context.get('repo_name')
         })
 
     return cc
 
 
+def get_headers_call_context(environ, strict=True):
+    if 'HTTP_X_RC_VCS_STREAM_CALL_CONTEXT' in environ:
+        packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
+        return msgpack.unpackb(packed_cc)
+    elif strict:
+        raise ValueError('Expected header HTTP_X_RC_VCS_STREAM_CALL_CONTEXT not found')
+
+
 class RequestWrapperTween(object):
     def __init__(self, handler, registry):
         self.handler = handler
         self.registry = registry
 
         # one-time configuration code goes here
 
     def __call__(self, request):
         start = time.time()
         log.debug('Starting request time measurement')
         response = None
 
         try:
             response = self.handler(request)
         finally:
             ua = get_user_agent(request.environ)
-            call_context = get_call_context(request.registry)
+            call_context = get_call_context(request)
             vcs_method = call_context.get('X-RC-Method', '_NO_VCS_METHOD')
             repo_name = call_context.get('X-RC-Repo-Name', '')
 
             count = request.request_count()
             _ver_ = vcsserver.__version__
             _path = safe_str(get_access_path(request.environ))
 
             ip = '127.0.0.1'
             match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
             resp_code = getattr(response, 'status_code', 'UNDEFINED')
 
             _view_path = f"{repo_name}@{_path}/{vcs_method}"
 
             total = time.time() - start
 
             log.info(
                 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
                 count, ip, request.environ.get('REQUEST_METHOD'),
                 _view_path, total, ua, _ver_,
                 extra={"time": total, "ver": _ver_, "code": resp_code,
                        "path": _path, "view_name": match_route, "user_agent": ua,
                        "vcs_method": vcs_method, "repo_name": repo_name}
             )
 
             statsd = request.registry.statsd
             if statsd:
                 match_route = request.matched_route.name if request.matched_route else _path
                 elapsed_time_ms = round(1000.0 * total)  # use ms only
                 statsd.timing(
                     "vcsserver_req_timing.histogram", elapsed_time_ms,
                     tags=[
                         "view_name:{}".format(match_route),
                         "code:{}".format(resp_code)
                     ],
                     use_decimals=False
                 )
                 statsd.incr(
                     "vcsserver_req_total", tags=[
                         "view_name:{}".format(match_route),
                         "code:{}".format(resp_code)
                     ])
 
         return response
 
 
 def includeme(config):
     config.add_tween(
         'vcsserver.tweens.request_wrapper.RequestWrapperTween',
     )