merged python3 into default

pyproject.toml

@@ -0,0 +1,54 b''
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rhodecode-vcsserver"
+description = "Version Control System Server for RhodeCode"
+authors = [
+    {name = "RhodeCode GmbH", email = "[e-mail]"},
+]
+
+license = {text = "GPL V3"}
+requires-python = ">=3.10"
+dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
+classifiers = [
+    'Development Status :: 6 - Mature',
+    'Intended Audience :: Developers',
+    'Operating System :: OS Independent',
+    'Topic :: Software Development :: Version Control',
+    'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
+    'Programming Language :: Python :: 3.10',
+]
+
+[project.entry-points."paste.app_factory"]
+main = "vcsserver.http_main:main"
+
+
+[tool.setuptools]
+packages = ["vcsserver"]
+
+[tool.setuptools.dynamic]
+readme = {file = ["README.rst"], content-type = "text/rst"}
+version = {file = "vcsserver/VERSION"}
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies.tests = {file = ["requirements_test.txt"]}
+
+[tool.ruff]
+select = [
+    # Pyflakes
+    "F",
+    # Pycodestyle
+    "E",
+    "W",
+    # isort
+    "I001"
+]
+ignore = [
+    "E501",  # line too long, handled by black
+]
+# Same as Black.
+line-length = 120
+
+[tool.ruff.isort]
+known-first-party = ["vcsserver"]

vcsserver/config/hooks.py

@@ -0,0 +1,27 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+HOOK_REPO_SIZE = 'changegroup.repo_size'
+
+# HG
+HOOK_PRE_PULL = 'preoutgoing.pre_pull'
+HOOK_PULL = 'outgoing.pull_logger'
+HOOK_PRE_PUSH = 'prechangegroup.pre_push'
+HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
+HOOK_PUSH = 'changegroup.push_logger'
+HOOK_PUSH_KEY = 'pushkey.key_push'

vcsserver/lib/logging_formatter.py

@@ -0,0 +1,53 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sys
+import logging
+
+
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
+
+# Sequences
+RESET_SEQ = "\033[0m"
+COLOR_SEQ = "\033[0;%dm"
+BOLD_SEQ = "\033[1m"
+
+COLORS = {
+    'CRITICAL': MAGENTA,
+    'ERROR': RED,
+    'WARNING': CYAN,
+    'INFO': GREEN,
+    'DEBUG': BLUE,
+    'SQL': YELLOW
+}
+
+
+class ColorFormatter(logging.Formatter):
+
+    def format(self, record):
+        """
+        Change record's levelname to use with COLORS enum
+        """
+        def_record = super().format(record)
+
+        levelname = record.levelname
+        start = COLOR_SEQ % (COLORS[levelname])
+        end = RESET_SEQ
+
+        colored_record = ''.join([start, def_record, end])
+        return colored_record

vcsserver/lib/rc_cache/archive_cache.py

@@ -0,0 +1,87 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import logging
+import os
+import diskcache
+from diskcache import RLock
+
+log = logging.getLogger(__name__)
+
+cache_meta = None
+
+
+class ReentrantLock(RLock):
+    def __enter__(self):
+        reentrant_lock_key = self._key
+
+        log.debug('Acquire ReentrantLock(key=%s) for archive cache generation...', reentrant_lock_key)
+        #self.acquire()
+        log.debug('Lock for key=%s acquired', reentrant_lock_key)
+
+    def __exit__(self, *exc_info):
+        #self.release()
+        pass
+
+
+def get_archival_config(config):
+
+    final_config = {
+        'archive_cache.eviction_policy': 'least-frequently-used'
+    }
+
+    for k, v in config.items():
+        if k.startswith('archive_cache'):
+            final_config[k] = v
+
+    return final_config
+
+
+def get_archival_cache_store(config):
+
+    global cache_meta
+    if cache_meta is not None:
+        return cache_meta
+
+    config = get_archival_config(config)
+
+    archive_cache_dir = config['archive_cache.store_dir']
+    archive_cache_size_gb = config['archive_cache.cache_size_gb']
+    archive_cache_shards = config['archive_cache.cache_shards']
+    archive_cache_eviction_policy = config['archive_cache.eviction_policy']
+
+    log.debug('Initializing archival cache instance under %s', archive_cache_dir)
+
+    # check if it's ok to write, and re-create the archive cache
+    if not os.path.isdir(archive_cache_dir):
+        os.makedirs(archive_cache_dir, exist_ok=True)
+
+    d_cache = diskcache.FanoutCache(
+        archive_cache_dir, shards=archive_cache_shards,
+        cull_limit=0,  # manual eviction required
+        size_limit=archive_cache_size_gb * 1024 * 1024 * 1024,
+        eviction_policy=archive_cache_eviction_policy,
+        timeout=30
+    )
+    cache_meta = d_cache
+    return cache_meta
+
+
+def includeme(config):
+    # init our cache at start, for vcsserver we don't init at runtime
+    # because our cache config is sent via wire on make archive call, this call just lazy-enables the client
+    return

vcsserver/lib/rc_json.py

@@ -0,0 +1,2 b''
+# use orjson by default
+import orjson as json

vcsserver/lib/svnremoterepo.py

@@ -0,0 +1,160 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import tempfile
+
+from svn import client
+from svn import core
+from svn import ra
+
+from mercurial import error
+
+from vcsserver.str_utils import safe_bytes
+
+core.svn_config_ensure(None)
+svn_config = core.svn_config_get_config(None)
+
+
+class RaCallbacks(ra.Callbacks):
+    @staticmethod
+    def open_tmp_file(pool):  # pragma: no cover
+        (fd, fn) = tempfile.mkstemp()
+        os.close(fd)
+        return fn
+
+    @staticmethod
+    def get_client_string(pool):
+        return b'RhodeCode-subversion-url-checker'
+
+
+class SubversionException(Exception):
+    pass
+
+
+class SubversionConnectionException(SubversionException):
+    """Exception raised when a generic error occurs when connecting to a repository."""
+
+
+def normalize_url(url):
+    if not url:
+        return url
+    if url.startswith(b'svn+http://') or url.startswith(b'svn+https://'):
+        url = url[4:]
+    url = url.rstrip(b'/')
+    return url
+
+
+def _create_auth_baton(pool):
+    """Create a Subversion authentication baton. """
+    # Give the client context baton a suite of authentication
+    # providers.h
+    platform_specific = [
+        'svn_auth_get_gnome_keyring_simple_provider',
+        'svn_auth_get_gnome_keyring_ssl_client_cert_pw_provider',
+        'svn_auth_get_keychain_simple_provider',
+        'svn_auth_get_keychain_ssl_client_cert_pw_provider',
+        'svn_auth_get_kwallet_simple_provider',
+        'svn_auth_get_kwallet_ssl_client_cert_pw_provider',
+        'svn_auth_get_ssl_client_cert_file_provider',
+        'svn_auth_get_windows_simple_provider',
+        'svn_auth_get_windows_ssl_server_trust_provider',
+    ]
+
+    providers = []
+
+    for p in platform_specific:
+        if getattr(core, p, None) is not None:
+            try:
+                providers.append(getattr(core, p)())
+            except RuntimeError:
+                pass
+
+    providers += [
+        client.get_simple_provider(),
+        client.get_username_provider(),
+        client.get_ssl_client_cert_file_provider(),
+        client.get_ssl_client_cert_pw_file_provider(),
+        client.get_ssl_server_trust_file_provider(),
+    ]
+
+    return core.svn_auth_open(providers, pool)
+
+
+class SubversionRepo(object):
+    """Wrapper for a Subversion repository.
+
+    It uses the SWIG Python bindings, see above for requirements.
+    """
+    def __init__(self, svn_url: bytes = b'', username: bytes = b'', password: bytes = b''):
+
+        self.username = username
+        self.password = password
+        self.svn_url = core.svn_path_canonicalize(svn_url)
+
+        self.auth_baton_pool = core.Pool()
+        self.auth_baton = _create_auth_baton(self.auth_baton_pool)
+        # self.init_ra_and_client() assumes that a pool already exists
+        self.pool = core.Pool()
+
+        self.ra = self.init_ra_and_client()
+        self.uuid = ra.get_uuid(self.ra, self.pool)
+
+    def init_ra_and_client(self):
+        """Initializes the RA and client layers, because sometimes getting
+        unified diffs runs the remote server out of open files.
+        """
+
+        if self.username:
+            core.svn_auth_set_parameter(self.auth_baton,
+                                        core.SVN_AUTH_PARAM_DEFAULT_USERNAME,
+                                        self.username)
+        if self.password:
+            core.svn_auth_set_parameter(self.auth_baton,
+                                        core.SVN_AUTH_PARAM_DEFAULT_PASSWORD,
+                                        self.password)
+
+        callbacks = RaCallbacks()
+        callbacks.auth_baton = self.auth_baton
+
+        try:
+            return ra.open2(self.svn_url, callbacks, svn_config, self.pool)
+        except SubversionException as e:
+            # e.child contains a detailed error messages
+            msglist = []
+            svn_exc = e
+            while svn_exc:
+                if svn_exc.args[0]:
+                    msglist.append(svn_exc.args[0])
+                svn_exc = svn_exc.child
+            msg = '\n'.join(msglist)
+            raise SubversionConnectionException(msg)
+
+
+class svnremoterepo(object):
+    """ the dumb wrapper for actual Subversion repositories """
+
+    def __init__(self, username: bytes = b'', password: bytes = b'', svn_url: bytes = b''):
+        self.username = username or b''
+        self.password = password or b''
+        self.path = normalize_url(svn_url)
+
+    def svn(self):
+        try:
+            return SubversionRepo(self.path, self.username, self.password)
+        except SubversionConnectionException as e:
+            raise error.Abort(safe_bytes(e))

vcsserver/remote/__init__.py

@@ -0,0 +1,17 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+

vcsserver/str_utils.py

@@ -0,0 +1,133 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import typing
+import base64
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+def safe_int(val, default=None) -> int:
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
+
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
+def base64_to_str(text) -> str:
+    return safe_str(base64.encodebytes(safe_bytes(text))).strip()
+
+
+def get_default_encodings() -> list[str]:
+    return ['utf8']
+
+
+def safe_str(str_, to_encoding=None) -> str:
+    """
+    safe str function. Does few trick to turn unicode_ into string
+
+    :param str_: str to encode
+    :param to_encoding: encode to this type UTF8 default
+    """
+    if isinstance(str_, str):
+        return str_
+
+    # if it's bytes cast to str
+    if not isinstance(str_, bytes):
+        return str(str_)
+
+    to_encoding = to_encoding or get_default_encodings()
+    if not isinstance(to_encoding, (list, tuple)):
+        to_encoding = [to_encoding]
+
+    for enc in to_encoding:
+        try:
+            return str(str_, enc)
+        except UnicodeDecodeError:
+            pass
+
+    return str(str_, to_encoding[0], 'replace')
+
+
+def safe_bytes(str_, from_encoding=None) -> bytes:
+    """
+    safe bytes function. Does few trick to turn str_ into bytes string:
+
+    :param str_: string to decode
+    :param from_encoding: encode from this type UTF8 default
+    """
+    if isinstance(str_, bytes):
+        return str_
+
+    if not isinstance(str_, str):
+        raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
+
+    from_encoding = from_encoding or get_default_encodings()
+    if not isinstance(from_encoding, (list, tuple)):
+        from_encoding = [from_encoding]
+
+    for enc in from_encoding:
+        try:
+            return str_.encode(enc)
+        except UnicodeDecodeError:
+            pass
+
+    return str_.encode(from_encoding[0], 'replace')
+
+
+def ascii_bytes(str_, allow_bytes=False) -> bytes:
+    """
+    Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
+    Fails with UnicodeError on invalid input.
+    This should be used where encoding and "safe" ambiguity should be avoided.
+    Where strings already have been encoded in other ways but still are unicode
+    string - for example to hex, base64, json, urlencoding, or are known to be
+    identifiers.
+    """
+    if allow_bytes and isinstance(str_, bytes):
+        return str_
+
+    if not isinstance(str_, str):
+        raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
+    return str_.encode('ascii')
+
+
+def ascii_str(str_) -> str:
+    """
+    Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
+    Fails with UnicodeError on invalid input.
+    This should be used where encoding and "safe" ambiguity should be avoided.
+    Where strings are encoded but also in other ways are known to be ASCII, and
+    where a unicode string is wanted without caring about encoding. For example
+    to hex, base64, urlencoding, or are known to be identifiers.
+    """
+
+    if not isinstance(str_, bytes):
+        raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
+    return str_.decode('ascii')

vcsserver/tests/test_utils.py

@@ -0,0 +1,53 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import pytest
+from vcsserver.str_utils import ascii_bytes, ascii_str
+
+
[e-mail]('given, expected', [
+    ('a', b'a'),
+    ('a', b'a'),
+])
+def test_ascii_bytes(given, expected):
+    assert ascii_bytes(given) == expected
+
+
[e-mail]('given', [
+    'å',
+    'å'.encode('utf8')
+])
+def test_ascii_bytes_raises(given):
+    with pytest.raises(ValueError):
+        ascii_bytes(given)
+
+
[e-mail]('given, expected', [
+    (b'a', 'a'),
+])
+def test_ascii_str(given, expected):
+    assert ascii_str(given) == expected
+
+
[e-mail]('given', [
+    'a',
+    'å'.encode('utf8'),
+    'å'
+])
+def test_ascii_str_raises(given):
+    with pytest.raises(ValueError):
+        ascii_str(given)

vcsserver/type_utils.py

@@ -0,0 +1,67 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def str2bool(str_):
+    """
+    returns True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param str_: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if str_ is None:
+        return False
+    if str_ in (True, False):
+        return str_
+    str_ = str(str_).strip().lower()
+    return str_ in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def aslist(obj, sep=None, strip=True) -> list:
+    """
+    Returns given string separated by sep as list
+
+    :param obj:
+    :param sep:
+    :param strip:
+    """
+    if isinstance(obj, str):
+        if obj in ['', ""]:
+            return []
+
+        lst = obj.split(sep)
+        if strip:
+            lst = [v.strip() for v in lst]
+        return lst
+    elif isinstance(obj, (list, tuple)):
+        return obj
+    elif obj is None:
+        return []
+    else:
+        return [obj]
+
+
+def assert_bytes(input_type, expected_types=(bytes,)):
+    if not isinstance(input_type, expected_types):
+        raise ValueError(f'input_types should be one of {expected_types} got {type(input_type)} instead')

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 4.28.0
+current_version = 5.0.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]

.hgignore

@@ -1,4 +1,5 b''
 syntax: glob
+
 *.orig
 *.pyc
 *.swp
@@ -19,8 +20,11 b' syntax: regexp'
 ^\.pydevproject$
 ^\.coverage$
 ^\.cache.*$
+^\.venv.*$
+^\.ruff_cache.*$
 ^\.rhodecode$
 
+
 ^.dev
 ^build/
 ^coverage\.xml$

Makefile

@@ -1,45 +1,111 b''
-.DEFAULT_GOAL := help
+# required for pushd to work..
+#SHELL = /bin/bash
+
 
 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 .PHONY: clean
-clean: ## full clean
+## Cleanup compiled and cache py files
+clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 
 
 .PHONY: test
-test: ## run test-clean and tests
+## run test-clean and tests
+test:
 	make test-clean
 	make test-only
 
 
-.PHONY:test-clean
-test-clean:  ## run test-clean and tests
+.PHONY: test-clean
+## run test-clean and tests
+test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 
 .PHONY: test-only
-test-only: ## run tests
+## Run tests only without cleanup
+test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
-	--cov=vcsserver --cov-report=term-missing --cov-report=html \
-	vcsserver
-
-
-.PHONY: generate-pkgs
-generate-pkgs: ## generate new python packages
-	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+	--cov-report=term-missing --cov-report=html \
+	--cov=vcsserver vcsserver
 
 
 .PHONY: pip-packages
-pip-packages: ## show outdated packages
+## Show outdated packages
+pip-packages:
 	python ${OUTDATED_PACKAGES}
 
 
-.PHONY: help
-help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}'
+.PHONY: build
+## Build sdist/egg
+build:
+	python -m build
+
+
+.PHONY: dev-env
+## make dev-env based on the requirements files and install develop of packages
+dev-env:
+	pip install build virtualenv
+	pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
+	pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_test.txt -r requirements_debug.txt
+	pip install -e .
+
+
+.PHONY: dev-srv
+## run develop server instance
+dev-srv:
+	pserve --reload .dev/dev.ini
+
+
+.PHONY: dev-srv-g
+## run gunicorn multi process workers
+dev-srv-g:
+	gunicorn --workers=4 --paste .dev/dev.ini --bind=0.0.0.0:10010 --worker-class=sync --threads=1 --config=configs/gunicorn_config.py --timeout=120
+
+# Default command on calling make
+.DEFAULT_GOAL := show-help
+
+.PHONY: show-help
+show-help:
+	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
+	@echo
+	@sed -n -e "/^## / { \
+		h; \
+		s/.*//; \
+		:doc" \
+		-e "H; \
+		n; \
+		s/^## //; \
+		t doc" \
+		-e "s/:.*//; \
+		G; \
+		s/\\n## /---/; \
+		s/\\n/ /g; \
+		p; \
+	}" ${MAKEFILE_LIST} \
+	| LC_ALL='C' sort --ignore-case \
+	| awk -F '---' \
+		-v ncol=$$(tput cols) \
+		-v indent=19 \
+		-v col_on="$$(tput setaf 6)" \
+		-v col_off="$$(tput sgr0)" \
+	'{ \
+		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
+		n = split($$2, words, " "); \
+		line_length = ncol - indent; \
+		for (i = 1; i <= n; i++) { \
+			line_length -= length(words[i]) + 1; \
+			if (line_length <= 0) { \
+				line_length = ncol - indent - length(words[i]) - 1; \
+				printf "\n%*s ", -indent, " "; \
+			} \
+			printf "%s ", words[i]; \
+		} \
+		printf "\n"; \
+	}'

configs/development.ini

@@ -1,4 +1,4 b''
-## -*- coding: utf-8 -*-
+#
 
 ; #################################
 ; RHODECODE VCSSERVER CONFIGURATION

configs/gunicorn_config.py

@@ -57,12 +57,9 b' def _get_process_rss(pid=None):'
 
 
 def _get_config(ini_path):
+    import configparser
 
     try:
-        import configparser
-    except ImportError:
-        import ConfigParser as configparser
-    try:
         config = configparser.RawConfigParser()
         config.read(ini_path)
         return config
@@ -336,7 +333,7 b' class RhodeCodeLogger(Logger):'
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
-        msecs = int((now - long(now)) * 1000)
+        msecs = int((now - int(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
     def atoms(self, resp, req, environ, request_time):

configs/production.ini

@@ -1,4 +1,4 b''
-## -*- coding: utf-8 -*-
+#
 
 ; #################################
 ; RHODECODE VCSSERVER CONFIGURATION

conftest.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,7 +39,7 b' def repeat(request):'
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = get_available_port()
-    print('Using vcsserver port %s' % (port, ))
+    print(f'Using vcsserver port {port}')
     return port
 
 

requirements.txt

@@ -1,47 +1,56 b''
-## dependencies
-
-# our custom configobj
-https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
+# deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
 
-dogpile.cache==0.9.0
-decorator==4.1.2
-dulwich==0.13.0
-hgsubversion==1.9.3
-hg-evolve==9.1.0
-mako==1.1.0
-markupsafe==1.1.1
-mercurial==5.1.1
-msgpack-python==0.5.6
-
-pastedeploy==2.1.0
-pyramid==1.10.4
-pyramid-mako==1.1.0
-pygit2==0.28.2
-
+async-timeout==4.0.2
+atomicwrites==1.4.1
+contextlib2==21.6.0
+cov-core==1.15.0
+  coverage==7.2.3
+diskcache==5.6.1
+dogpile.cache==1.2.2
+  decorator==5.1.1
+  stevedore==5.0.0
+    pbr==5.11.1
+dulwich==0.21.5
+  urllib3==1.26.14
+gunicorn==21.0.1
+  packaging==23.1
+hg-evolve==11.0.2
+importlib-metadata==6.0.0
+  zipp==3.15.0
+mercurial==6.3.3
+mock==5.0.2
+more-itertools==9.1.0
+msgpack==1.0.5
+orjson==3.9.2
+psutil==5.9.5
+py==1.11.0
+pygit2==1.12.2
+  cffi==1.15.1
+    pycparser==2.21
+pygments==2.15.1
+pyparsing==3.0.9
+pyramid==2.0.1
+  hupper==1.12
+  plaster==1.1.2
+  plaster-pastedeploy==1.0.1
+    pastedeploy==3.0.1
+    plaster==1.1.2
+  translationstring==1.4
+  venusian==3.0.0
+  webob==1.8.7
+  zope.deprecation==5.0.0
+  zope.interface==6.0.0
+redis==4.6.0
+  async-timeout==4.0.2
 repoze.lru==0.7
-redis==3.5.3
-simplejson==3.16.0
-subprocess32==3.5.4
-subvertpy==0.10.1
+scandir==1.10.0
+setproctitle==1.3.2
+subvertpy==0.11.0
+wcwidth==0.2.6
 
-six==1.11.0
-translationstring==1.3
-webob==1.8.5
-zope.deprecation==4.4.0
-zope.interface==4.6.0
-
-## http servers
-gevent==1.5.0
-greenlet==0.4.15
-gunicorn==19.9.0
-waitress==1.3.1
-
-## debug
-ipdb==0.13.2
-ipython==5.1.0
 
 ## test related requirements
--r requirements_test.txt
+#-r requirements_test.txt
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_debug.txt

@@ -1,8 +1,22 b''
 ## special libraries we could extend the requirements.txt file with to add some
-## custom libraries useful for debug and memory tracing
-
-## uncomment inclusion of this file in requirements.txt run make generate-pkgs and nix-shell
+## custom libraries usefull for debug and memory tracing
 
 objgraph
 memory-profiler
 pympler
+
+## debug
+ipdb
+ipython
+rich
+
+# format
+flake8
+ruff
+
+pipdeptree==2.7.1
+invoke==2.0.0
+bumpversion==0.6.0
+bump2version==1.0.1
+
+docutils-stubs

requirements_test.txt

@@ -1,16 +1,45 b''
 # test related requirements
-pytest==4.6.9
-py==1.8.1
-pytest-cov==2.8.1
-pytest-sugar==0.9.3
-pytest-runner==5.2.0
+
+cov-core==1.15.0
+  coverage==7.2.3
+mock==5.0.2
+py==1.11.0
+pytest-cov==4.0.0
+  coverage==7.2.3
+  pytest==7.3.1
+    attrs==22.2.0
+    iniconfig==2.0.0
+    packaging==23.1
+    pluggy==1.0.0
 pytest-profiling==1.7.0
-pytest-timeout==1.3.3
-gprof2dot==2017.9.19
+  gprof2dot==2022.7.29
+  pytest==7.3.1
+    attrs==22.2.0
+    iniconfig==2.0.0
+    packaging==23.1
+    pluggy==1.0.0
+  six==1.16.0
+pytest-runner==6.0.0
+pytest-sugar==0.9.7
+  packaging==23.1
+  pytest==7.3.1
+    attrs==22.2.0
+    iniconfig==2.0.0
+    packaging==23.1
+    pluggy==1.0.0
+  termcolor==2.3.0
+pytest-timeout==2.1.0
+  pytest==7.3.1
+    attrs==22.2.0
+    iniconfig==2.0.0
+    packaging==23.1
+    pluggy==1.0.0
+webtest==3.0.0
+  beautifulsoup4==4.11.2
+    soupsieve==2.4
+  waitress==2.1.2
+  webob==1.8.7
 
-mock==3.0.5
-cov-core==1.15.0
-coverage==4.5.4
-
-webtest==2.0.34
-beautifulsoup4==4.6.3
+# RhodeCode test-data
+rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
+rc_testdata==0.11.0

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.28.0
  No newline at end of file
+5.0.0
  No newline at end of file

vcsserver/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 b''
 import pkgutil
 
 
-__version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
+__version__ = pkgutil.get_data('vcsserver', 'VERSION').strip().decode()
 
 # link to config for pyramid
 CONFIG = {}

vcsserver/base.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,13 +16,18 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 import os
 import sys
+import tempfile
 import traceback
 import logging
-import urlparse
+import urllib.parse
+
+from vcsserver.lib.rc_cache.archive_cache import get_archival_cache_store
+from vcsserver.lib.rc_cache import region_meta
 
 from vcsserver import exceptions
 from vcsserver.exceptions import NoContentException
-from vcsserver.hgcompat import (archival)
+from vcsserver.hgcompat import archival
+from vcsserver.str_utils import safe_bytes
 
 log = logging.getLogger(__name__)
 
@@ -37,7 +42,7 b' class RepoFactory(object):'
     repo_type = None
 
     def __init__(self):
-        pass
+        self._cache_region = region_meta.dogpile_cache_regions['repo_object']
 
     def _create_config(self, path, config):
         config = {}
@@ -55,30 +60,33 b' def obfuscate_qs(query_string):'
         return None
 
     parsed = []
-    for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
+    for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
         if k in ['auth_token', 'api_key']:
             v = "*****"
         parsed.append((k, v))
 
     return '&'.join('{}{}'.format(
-        k, '={}'.format(v) if v else '') for k, v in parsed)
+        k, f'={v}' if v else '') for k, v in parsed)
 
 
-def raise_from_original(new_type):
+def raise_from_original(new_type, org_exc: Exception):
     """
     Raise a new exception type with original args and traceback.
     """
+
     exc_type, exc_value, exc_traceback = sys.exc_info()
     new_exc = new_type(*exc_value.args)
+
     # store the original traceback into the new exc
-    new_exc._org_exc_tb = traceback.format_exc(exc_traceback)
+    new_exc._org_exc_tb = traceback.format_tb(exc_traceback)
 
     try:
-        raise new_exc, None, exc_traceback
+        raise new_exc.with_traceback(exc_traceback)
     finally:
         del exc_traceback
 
 
+
 class ArchiveNode(object):
     def __init__(self, path, mode, is_link, raw_bytes):
         self.path = path
@@ -87,34 +95,58 b' class ArchiveNode(object):'
         self.raw_bytes = raw_bytes
 
 
-def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
-                 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
+def store_archive_in_cache(node_walker, archive_key, kind, mtime, archive_at_path, archive_dir_name,
+                           commit_id, write_metadata=True, extra_metadata=None, cache_config=None):
     """
+    Function that would store an generate archive and send it to a dedicated backend store
+    In here we use diskcache
+
+    :param node_walker: a generator returning nodes to add to archive
+    :param archive_key: key used to store the path
+    :param kind: archive kind
+    :param mtime: time of creation
+    :param archive_at_path: default '/' the path at archive was started. if this is not '/' it means it's a partial archive
+    :param archive_dir_name: inside dir name when creating an archive
+    :param commit_id: commit sha of revision archive was created at
+    :param write_metadata:
+    :param extra_metadata:
+    :param cache_config:
+
     walker should be a file walker, for example:
-        def walker():
+        def node_walker():
             for file_info in files:
                 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
     """
     extra_metadata = extra_metadata or {}
 
+    d_cache = get_archival_cache_store(config=cache_config)
+
+    if archive_key in d_cache:
+        with d_cache as d_cache_reader:
+            reader, tag = d_cache_reader.get(archive_key, read=True, tag=True, retry=True)
+            return reader.name
+
+    archive_tmp_path = safe_bytes(tempfile.mkstemp()[1])
+    log.debug('Creating new temp archive in %s', archive_tmp_path)
+
     if kind == "tgz":
-        archiver = archival.tarit(archive_dest_path, mtime, "gz")
+        archiver = archival.tarit(archive_tmp_path, mtime, b"gz")
     elif kind == "tbz2":
-        archiver = archival.tarit(archive_dest_path, mtime, "bz2")
+        archiver = archival.tarit(archive_tmp_path, mtime, b"bz2")
     elif kind == 'zip':
-        archiver = archival.zipit(archive_dest_path, mtime)
+        archiver = archival.zipit(archive_tmp_path, mtime)
     else:
         raise exceptions.ArchiveException()(
-            'Remote does not support: "%s" archive type.' % kind)
+            f'Remote does not support: "{kind}" archive type.')
 
-    for f in walker(commit_id, archive_at_path):
-        f_path = os.path.join(archive_dir_name, f.path.lstrip('/'))
+    for f in node_walker(commit_id, archive_at_path):
+        f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
         try:
             archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
         except NoContentException:
             # NOTE(marcink): this is a special case for SVN so we can create "empty"
             # directories which arent supported by archiver
-            archiver.addfile(os.path.join(f_path, '.dir'), f.mode, f.is_link, '')
+            archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
 
     if write_metadata:
         metadata = dict([
@@ -123,8 +155,41 b' def archive_repo(walker, archive_dest_pa'
         ])
         metadata.update(extra_metadata)
 
-        meta = ["%s:%s" % (f_name, value) for f_name, value in metadata.items()]
-        f_path = os.path.join(archive_dir_name, '.archival.txt')
-        archiver.addfile(f_path, 0o644, False, '\n'.join(meta))
+        meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
+        f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
+        archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
+
+    archiver.done()
+
+    # ensure set & get are atomic
+    with d_cache.transact():
+
+        with open(archive_tmp_path, 'rb') as archive_file:
+            add_result = d_cache.set(archive_key, archive_file, read=True, tag='db-name', retry=True)
+            if not add_result:
+                log.error('Failed to store cache for key=%s', archive_key)
+
+        os.remove(archive_tmp_path)
 
-    return archiver.done()
+        reader, tag = d_cache.get(archive_key, read=True, tag=True, retry=True)
+        if not reader:
+            raise AssertionError(f'empty reader on key={archive_key} added={add_result}')
+
+        return reader.name
+
+
+class BinaryEnvelope(object):
+    def __init__(self, val):
+        self.val = val
+
+
+class BytesEnvelope(bytes):
+    def __new__(cls, content):
+        if isinstance(content, bytes):
+            return super().__new__(cls, content)
+        else:
+            raise TypeError('Content must be bytes.')
+
+
+class BinaryBytesEnvelope(BytesEnvelope):
+    pass

vcsserver/config/__init__.py

@@ -0,0 +1,1 b''
+# Copyright (C) 2014-2023 RhodeCode GmbH

vcsserver/config/settings_maker.py

@@ -1,6 +1,4 b''
-# -*- coding: utf-8 -*-
-
-# Copyright (C) 2010-2020 RhodeCode GmbH
+# Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -25,6 +23,9 b' import functools'
 import logging
 import tempfile
 import logging.config
+
+from vcsserver.type_utils import str2bool, aslist
+
 log = logging.getLogger(__name__)
 
 # skip keys, that are set here, so we don't double process those
@@ -33,47 +34,6 b' set_keys = {'
 }
 
 
-def str2bool(_str):
-    """
-    returns True/False value from given string, it tries to translate the
-    string into boolean
-
-    :param _str: string value to translate into boolean
-    :rtype: boolean
-    :returns: boolean from given string
-    """
-    if _str is None:
-        return False
-    if _str in (True, False):
-        return _str
-    _str = str(_str).strip().lower()
-    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
-
-
-def aslist(obj, sep=None, strip=True):
-    """
-    Returns given string separated by sep as list
-
-    :param obj:
-    :param sep:
-    :param strip:
-    """
-    if isinstance(obj, (basestring,)):
-        if obj in ['', ""]:
-            return []
-
-        lst = obj.split(sep)
-        if strip:
-            lst = [v.strip() for v in lst]
-        return lst
-    elif isinstance(obj, (list, tuple)):
-        return obj
-    elif obj is None:
-        return []
-    else:
-        return [obj]
-
-
 class SettingsMaker(object):
 
     def __init__(self, app_settings):
@@ -81,8 +41,9 b' class SettingsMaker(object):'
 
     @classmethod
     def _bool_func(cls, input_val):
-        if isinstance(input_val, unicode):
-            input_val = input_val.encode('utf8')
+        if isinstance(input_val, bytes):
+            # decode to str
+            input_val = input_val.decode('utf8')
         return str2bool(input_val)
 
     @classmethod
@@ -108,10 +69,10 b' class SettingsMaker(object):'
 
         # ensure we have our dir created
         if not os.path.isdir(input_val) and ensure_dir:
-            os.makedirs(input_val, mode=mode)
+            os.makedirs(input_val, mode=mode, exist_ok=True)
 
         if not os.path.isdir(input_val):
-            raise Exception('Dir at {} does not exist'.format(input_val))
+            raise Exception(f'Dir at {input_val} does not exist')
         return input_val
 
     @classmethod
@@ -163,7 +124,7 b' class SettingsMaker(object):'
                       'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
             return
 
-        with open(logging_conf, 'rb') as f:
+        with open(logging_conf, 'rt') as f:
             ini_template = textwrap.dedent(f.read())
             ini_template = string.Template(ini_template).safe_substitute(
                 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,

vcsserver/echo_stub/__init__.py

@@ -1,3 +1,5 b''
+# Copyright (C) 2014-2023 RhodeCode GmbH
+
 """
 Provides a stub implementation for VCS operations.
 

vcsserver/echo_stub/echo_app.py

@@ -1,3 +1,5 b''
+# Copyright (C) 2014-2023 RhodeCode GmbH
+
 """
 Implementation of :class:`EchoApp`.
 
@@ -23,7 +25,7 b' class EchoApp(object):'
         status = '200 OK'
         headers = [('Content-Type', 'text/plain')]
         start_response(status, headers)
-        return ["ECHO"]
+        return [b"ECHO"]
 
 
 class EchoAppStream(object):
@@ -41,8 +43,8 b' class EchoAppStream(object):'
         start_response(status, headers)
 
         def generator():
-            for _ in xrange(1000000):
-                yield "ECHO"
+            for _ in range(1000000):
+                yield b"ECHO_STREAM"
         return generator()
 
 

vcsserver/echo_stub/remote_wsgi.py

@@ -1,3 +1,5 b''
+# Copyright (C) 2014-2023 RhodeCode GmbH
+
 """
 Provides the same API as :mod:`remote_wsgi`.
 
@@ -42,4 +44,4 b' def _assert_valid_config(config):'
     config = config.copy()
 
     # This is what git needs from config at this stage
-    config.pop('git_update_server_info')
+    config.pop(b'git_update_server_info')

vcsserver/exceptions.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -109,7 +109,7 b' class HTTPRepoLocked(HTTPLocked):'
     def __init__(self, title, status_code=None, **kwargs):
         self.code = status_code or HTTPLocked.code
         self.title = title
-        super(HTTPRepoLocked, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
 
 class HTTPRepoBranchProtected(HTTPForbidden):

vcsserver/git_lfs/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,4 +16,4 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
-from app import create_app
+from .app import create_app

vcsserver/git_lfs/app.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,16 +19,16 b' import re'
 import logging
 from wsgiref.util import FileWrapper
 
-import simplejson as json
 from pyramid.config import Configurator
 from pyramid.response import Response, FileIter
 from pyramid.httpexceptions import (
     HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
     HTTPUnprocessableEntity)
 
+from vcsserver.lib.rc_json import json
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
-from vcsserver.utils import safe_int
+from vcsserver.str_utils import safe_int
 
 log = logging.getLogger(__name__)
 
@@ -116,7 +116,7 b' def lfs_objects_batch(request):'
 
         obj_data = {'oid': oid}
 
-        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid, 
+        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
                                      _scheme=http_scheme)
         obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
                                             _scheme=http_scheme)
@@ -212,7 +212,7 b' def lfs_objects_verify(request):'
 
     store_size = store.size_oid()
     if store_size != size:
-        msg = 'requested file size mismatch store size:%s requested:%s' % (
+        msg = 'requested file size mismatch store size:{} requested:{}'.format(
             store_size, size)
         return write_response_error(
             HTTPUnprocessableEntity, msg)

vcsserver/git_lfs/lib.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -51,7 +51,7 b' class OidHandler(object):'
 
         if not store.has_oid():
             # error reply back to client that something is wrong with dl
-            err_msg = 'object: {} does not exist in store'.format(store.oid)
+            err_msg = f'object: {store.oid} does not exist in store'
             has_errors = OrderedDict(
                 error=OrderedDict(
                     code=404,

vcsserver/git_lfs/tests/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,8 +18,9 b''
 import os
 import pytest
 from webtest.app import TestApp as WebObTestApp
-import simplejson as json
 
+from vcsserver.lib.rc_json import json
+from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.app import create_app
 
 
@@ -54,19 +55,19 b' class TestLFSApplication(object):'
     def test_app_deprecated_endpoint(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
         assert response.status_code == 501
-        assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
+        assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
 
     def test_app_lock_verify_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
-            u'message': u'GIT LFS locking api not supported'}
+            'message': 'GIT LFS locking api not supported'}
 
     def test_app_lock_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
-            u'message': u'GIT LFS locking api not supported'}
+            'message': 'GIT LFS locking api not supported'}
 
     def test_app_batch_api_missing_auth(self, git_lfs_app):
         git_lfs_app.post_json(
@@ -77,14 +78,14 b' class TestLFSApplication(object):'
             '/repo/info/lfs/objects/batch', params={}, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
-            u'message': u'unsupported operation mode: `None`'}
+            'message': 'unsupported operation mode: `None`'}
 
     def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={'operation': 'download'},
             status=400, extra_environ=http_auth)
         assert json.loads(response.text) == {
-            u'message': u'missing objects data'}
+            'message': 'missing objects data'}
 
     def test_app_batch_api_unsupported_data_in_objects(
             self, git_lfs_app, http_auth):
@@ -94,7 +95,7 b' class TestLFSApplication(object):'
             '/repo/info/lfs/objects/batch', params=params, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
-            u'message': u'unsupported data in objects'}
+            'message': 'unsupported data in objects'}
 
     def test_app_batch_api_download_missing_object(
             self, git_lfs_app, http_auth):
@@ -105,12 +106,12 b' class TestLFSApplication(object):'
             extra_environ=http_auth)
 
         expected_objects = [
-            {u'authenticated': True,
-             u'errors': {u'error': {
-                 u'code': 404,
-                 u'message': u'object: 123 does not exist in store'}},
-             u'oid': u'123',
-             u'size': u'1024'}
+            {'authenticated': True,
+             'errors': {'error': {
+                 'code': 404,
+                 'message': 'object: 123 does not exist in store'}},
+             'oid': '123',
+             'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
@@ -121,7 +122,7 b' class TestLFSApplication(object):'
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
-            f.write('OID_CONTENT')
+            f.write(safe_bytes('OID_CONTENT'))
 
         params = {'operation': 'download',
                   'objects': [{'oid': oid, 'size': '1024'}]}
@@ -130,14 +131,14 b' class TestLFSApplication(object):'
             extra_environ=http_auth)
 
         expected_objects = [
-            {u'authenticated': True,
-             u'actions': {
-                 u'download': {
-                     u'header': {u'Authorization': u'Basic XXXXX'},
-                     u'href': u'http://localhost/repo/info/lfs/objects/456'},
+            {'authenticated': True,
+             'actions': {
+                 'download': {
+                     'header': {'Authorization': 'Basic XXXXX'},
+                     'href': 'http://localhost/repo/info/lfs/objects/456'},
              },
-             u'oid': u'456',
-             u'size': u'1024'}
+             'oid': '456',
+             'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
@@ -149,18 +150,18 b' class TestLFSApplication(object):'
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
-            {u'authenticated': True,
-             u'actions': {
-                 u'upload': {
-                     u'header': {u'Authorization': u'Basic XXXXX',
-                                 u'Transfer-Encoding': u'chunked'},
-                     u'href': u'http://localhost/repo/info/lfs/objects/123'},
-                 u'verify': {
-                     u'header': {u'Authorization': u'Basic XXXXX'},
-                     u'href': u'http://localhost/repo/info/lfs/verify'}
+            {'authenticated': True,
+             'actions': {
+                 'upload': {
+                     'header': {'Authorization': 'Basic XXXXX',
+                                 'Transfer-Encoding': 'chunked'},
+                     'href': 'http://localhost/repo/info/lfs/objects/123'},
+                 'verify': {
+                     'header': {'Authorization': 'Basic XXXXX'},
+                     'href': 'http://localhost/repo/info/lfs/verify'}
              },
-             u'oid': u'123',
-             u'size': u'1024'}
+             'oid': '123',
+             'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
@@ -172,18 +173,18 b' class TestLFSApplication(object):'
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
-            {u'authenticated': True,
-             u'actions': {
-                 u'upload': {
-                     u'header': {u'Authorization': u'Basic XXXXX',
-                                 u'Transfer-Encoding': u'chunked'},
-                     u'href': u'https://localhost/repo/info/lfs/objects/123'},
-                 u'verify': {
-                     u'header': {u'Authorization': u'Basic XXXXX'},
-                     u'href': u'https://localhost/repo/info/lfs/verify'}
+            {'authenticated': True,
+             'actions': {
+                 'upload': {
+                     'header': {'Authorization': 'Basic XXXXX',
+                                 'Transfer-Encoding': 'chunked'},
+                     'href': 'https://localhost/repo/info/lfs/objects/123'},
+                 'verify': {
+                     'header': {'Authorization': 'Basic XXXXX'},
+                     'href': 'https://localhost/repo/info/lfs/verify'}
              },
-             u'oid': u'123',
-             u'size': u'1024'}
+             'oid': '123',
+             'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
@@ -195,7 +196,7 b' class TestLFSApplication(object):'
             status=400)
 
         assert json.loads(response.text) == {
-            u'message': u'missing oid and size in request data'}
+            'message': 'missing oid and size in request data'}
 
     def test_app_verify_api_missing_obj(self, git_lfs_app):
         params = {'oid': 'missing', 'size': '1024'}
@@ -204,7 +205,7 b' class TestLFSApplication(object):'
             status=404)
 
         assert json.loads(response.text) == {
-            u'message': u'oid `missing` does not exists in store'}
+            'message': 'oid `missing` does not exists in store'}
 
     def test_app_verify_api_size_mismatch(self, git_lfs_app):
         oid = 'existing'
@@ -212,15 +213,15 b' class TestLFSApplication(object):'
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
-            f.write('OID_CONTENT')
+            f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params, status=422)
 
         assert json.loads(response.text) == {
-            u'message': u'requested file size mismatch '
-                        u'store size:11 requested:1024'}
+            'message': 'requested file size mismatch '
+                        'store size:11 requested:1024'}
 
     def test_app_verify_api(self, git_lfs_app):
         oid = 'existing'
@@ -228,14 +229,14 b' class TestLFSApplication(object):'
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
-            f.write('OID_CONTENT')
+            f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': 11}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params)
 
         assert json.loads(response.text) == {
-            u'message': {u'size': u'ok', u'in_store': u'ok'}}
+            'message': {'size': 'ok', 'in_store': 'ok'}}
 
     def test_app_download_api_oid_not_existing(self, git_lfs_app):
         oid = 'missing'
@@ -244,7 +245,7 b' class TestLFSApplication(object):'
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
 
         assert json.loads(response.text) == {
-            u'message': u'requested file with oid `missing` not found in store'}
+            'message': 'requested file with oid `missing` not found in store'}
 
     def test_app_download_api(self, git_lfs_app):
         oid = 'existing'
@@ -252,7 +253,7 b' class TestLFSApplication(object):'
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
-            f.write('OID_CONTENT')
+            f.write(safe_bytes('OID_CONTENT'))
 
         response = git_lfs_app.get(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid))
@@ -264,7 +265,7 b' class TestLFSApplication(object):'
         response = git_lfs_app.put(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
 
-        assert json.loads(response.text) == {u'upload': u'ok'}
+        assert json.loads(response.text) == {'upload': 'ok'}
 
         # verify that we actually wrote that OID
         oid_path = os.path.join(git_lfs_app._store, oid)

vcsserver/git_lfs/tests/test_lib.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,6 +17,7 b''
 
 import os
 import pytest
+from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 
 
@@ -70,7 +71,7 b' class TestOidHandler(object):'
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
-            f.write('CONTENT')
+            f.write(safe_bytes('CONTENT'))
 
         response, has_errors = oid_handler.exec_operation('download')
 
@@ -86,7 +87,7 b' class TestOidHandler(object):'
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
-            f.write('CONTENT')
+            f.write(safe_bytes('CONTENT'))
         oid_handler.obj_size = 7
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
@@ -98,7 +99,7 b' class TestOidHandler(object):'
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
-            f.write('CONTENT')
+            f.write(safe_bytes('CONTENT'))
 
         oid_handler.obj_size = 10240
         response, has_errors = oid_handler.exec_operation('upload')
@@ -127,7 +128,7 b' class TestLFSStore(object):'
 
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
-            f.write('CONTENT')
+            f.write(safe_bytes('CONTENT'))
 
         assert os.path.isfile(oid_location)
 
@@ -136,6 +137,6 b' class TestLFSStore(object):'
         assert lfs_store.has_oid() is False
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
-            f.write('CONTENT')
+            f.write(safe_bytes('CONTENT'))
 
-        assert lfs_store.has_oid() is True
  No newline at end of file
+        assert lfs_store.has_oid() is True

vcsserver/git_lfs/utils.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/hgcompat.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,8 +21,11 b' Mercurial libs compatibility'
 
 import mercurial
 from mercurial import demandimport
+
 # patch demandimport, due to bug in mercurial when it always triggers
 # demandimport.enable()
+from vcsserver.str_utils import safe_bytes
+
 demandimport.enable = lambda *args, **kwargs: 1
 
 from mercurial import ui
@@ -39,7 +42,8 b' from mercurial import subrepo'
 from mercurial import subrepoutil
 from mercurial import tags as hg_tag
 from mercurial import util as hgutil
-from mercurial.commands import clone, nullid, pull
+from mercurial.commands import clone, pull
+from mercurial.node import nullid
 from mercurial.context import memctx, memfilectx
 from mercurial.error import (
     LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
@@ -53,7 +57,7 b' from mercurial.encoding import tolocal'
 from mercurial.discovery import findcommonoutgoing
 from mercurial.hg import peer
 from mercurial.httppeer import makepeer
-from mercurial.util import url as hg_url
+from mercurial.utils.urlutil import url as hg_url
 from mercurial.scmutil import revrange, revsymbol
 from mercurial.node import nullrev
 from mercurial import exchange
@@ -63,8 +67,13 b' from hgext import largefiles'
 # infinit looping when given invalid resources
 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
 
+# hg strip is in core now
+from mercurial import strip as hgext_strip
+
 
 def get_ctx(repo, ref):
+    if not isinstance(ref, int):
+        ref = safe_bytes(ref)
     try:
         ctx = repo[ref]
     except (ProgrammingError, TypeError):
@@ -73,7 +82,7 b' def get_ctx(repo, ref):'
         ctx = revsymbol(repo, ref)
     except (LookupError, RepoLookupError):
         # Similar case as above but only for refs that are not numeric
-        if isinstance(ref, (int, long)):
+        if isinstance(ref, int):
             raise
         ctx = revsymbol(repo, ref)
     return ctx

vcsserver/hgpatches.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -62,7 +62,7 b' def _dynamic_capabilities_wrapper(lfprot'
 
 def patch_subrepo_type_mapping():
     from collections import defaultdict
-    from hgcompat import subrepo, subrepoutil
+    from .hgcompat import subrepo, subrepoutil
     from vcsserver.exceptions import SubrepoMergeException
 
     class NoOpSubrepo(subrepo.abstractsubrepo):

vcsserver/hook_utils/__init__.py

@@ -1,7 +1,5 b''
-# -*- coding: utf-8 -*-
-
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,6 +23,7 b' import logging'
 import pkg_resources
 
 import vcsserver
+from vcsserver.str_utils import safe_bytes
 
 log = logging.getLogger(__name__)
 
@@ -49,7 +48,7 b' def install_git_hooks(repo_path, bare, e'
     hooks_path = get_git_hooks_path(repo_path, bare)
 
     if not os.path.isdir(hooks_path):
-        os.makedirs(hooks_path, mode=0o777)
+        os.makedirs(hooks_path, mode=0o777, exist_ok=True)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
@@ -70,14 +69,13 b' def install_git_hooks(repo_path, bare, e'
             log.debug('writing git %s hook file at %s !', h_type, _hook_file)
             try:
                 with open(_hook_file, 'wb') as f:
-                    template = template.replace(
-                        '_TMPL_', vcsserver.__version__)
-                    template = template.replace('_DATE_', timestamp)
-                    template = template.replace('_ENV_', executable)
-                    template = template.replace('_PATH_', path)
+                    template = template.replace(b'_TMPL_', safe_bytes(vcsserver.__version__))
+                    template = template.replace(b'_DATE_', safe_bytes(timestamp))
+                    template = template.replace(b'_ENV_', safe_bytes(executable))
+                    template = template.replace(b'_PATH_', safe_bytes(path))
                     f.write(template)
                 os.chmod(_hook_file, 0o755)
-            except IOError:
+            except OSError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
@@ -102,7 +100,7 b' def install_svn_hooks(repo_path, executa'
     executable = executable or sys.executable
     hooks_path = get_svn_hooks_path(repo_path)
     if not os.path.isdir(hooks_path):
-        os.makedirs(hooks_path, mode=0o777)
+        os.makedirs(hooks_path, mode=0o777, exist_ok=True)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
@@ -124,15 +122,14 b' def install_svn_hooks(repo_path, executa'
 
             try:
                 with open(_hook_file, 'wb') as f:
-                    template = template.replace(
-                        '_TMPL_', vcsserver.__version__)
-                    template = template.replace('_DATE_', timestamp)
-                    template = template.replace('_ENV_', executable)
-                    template = template.replace('_PATH_', path)
+                    template = template.replace(b'_TMPL_', safe_bytes(vcsserver.__version__))
+                    template = template.replace(b'_DATE_', safe_bytes(timestamp))
+                    template = template.replace(b'_ENV_', safe_bytes(executable))
+                    template = template.replace(b'_PATH_', safe_bytes(path))
 
                     f.write(template)
                 os.chmod(_hook_file, 0o755)
-            except IOError:
+            except OSError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
@@ -141,16 +138,16 b' def install_svn_hooks(repo_path, executa'
 
 
 def get_version_from_hook(hook_path):
-    version = ''
+    version = b''
     hook_content = read_hook_content(hook_path)
-    matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
+    matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
     if matches:
         try:
             version = matches.groups()[0]
             log.debug('got version %s from hooks.', version)
         except Exception:
             log.exception("Exception while reading the hook version.")
-    return version.replace("'", "")
+    return version.replace(b"'", b"")
 
 
 def check_rhodecode_hook(hook_path):
@@ -169,8 +166,8 b' def check_rhodecode_hook(hook_path):'
     return False
 
 
-def read_hook_content(hook_path):
-    content = ''
+def read_hook_content(hook_path) -> bytes:
+    content = b''
     if os.path.isfile(hook_path):
         with open(hook_path, 'rb') as f:
             content = f.read()

vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl

@@ -11,7 +11,7 b' try:'
 except ImportError:
     if os.environ.get('RC_DEBUG_GIT_HOOK'):
         import traceback
-        print traceback.format_exc()
+        print(traceback.format_exc())
     hooks = None
 
 
@@ -42,7 +42,7 b' def main():'
         # TODO: johbo: Improve handling of this special case
         if not getattr(error, '_vcs_kind', None) == 'repo_locked':
             raise
-        print 'ERROR:', error
+        print(f'ERROR: {error}')
         sys.exit(1)
     sys.exit(0)
 

vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl

@@ -11,7 +11,7 b' try:'
 except ImportError:
     if os.environ.get('RC_DEBUG_GIT_HOOK'):
         import traceback
-        print traceback.format_exc()
+        print(traceback.format_exc())
     hooks = None
 
 
@@ -42,7 +42,7 b' def main():'
         # TODO: johbo: Improve handling of this special case
         if not getattr(error, '_vcs_kind', None) == 'repo_locked':
             raise
-        print 'ERROR:', error
+        print(f'ERROR: {error}')
         sys.exit(1)
     sys.exit(0)
 

vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl

@@ -12,7 +12,7 b' try:'
 except ImportError:
     if os.environ.get('RC_DEBUG_SVN_HOOK'):
         import traceback
-        print traceback.format_exc()
+        print(traceback.format_exc())
     hooks = None
 
 
@@ -40,7 +40,7 b' def main():'
         # TODO: johbo: Improve handling of this special case
         if not getattr(error, '_vcs_kind', None) == 'repo_locked':
             raise
-        print 'ERROR:', error
+        print(f'ERROR: {error}')
         sys.exit(1)
     sys.exit(0)
 

vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl

@@ -12,7 +12,7 b' try:'
 except ImportError:
     if os.environ.get('RC_DEBUG_SVN_HOOK'):
         import traceback
-        print traceback.format_exc()
+        print(traceback.format_exc())
     hooks = None
 
 
@@ -43,7 +43,7 b' def main():'
         # TODO: johbo: Improve handling of this special case
         if not getattr(error, '_vcs_kind', None) == 'repo_locked':
             raise
-        print 'ERROR:', error
+        print(f'ERROR: {error}')
         sys.exit(1)
     sys.exit(0)
 

vcsserver/hooks.py

@@ -1,7 +1,5 b''
-# -*- coding: utf-8 -*-
-
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,51 +22,70 b' import logging'
 import collections
 import importlib
 import base64
+import msgpack
+import dataclasses
+import pygit2
 
-from httplib import HTTPConnection
+import http.client
 
 
 import mercurial.scmutil
 import mercurial.node
-import simplejson as json
 
+from vcsserver.lib.rc_json import json
 from vcsserver import exceptions, subprocessio, settings
+from vcsserver.str_utils import ascii_str, safe_str
+from vcsserver.remote.git import Repository
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
+    proto = 'msgpack.v1'
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
+    def __repr__(self):
+        return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
+
     def __call__(self, method, extras):
-        connection = HTTPConnection(self.hooks_uri)
-        body = self._serialize(method, extras)
-        try:
-            connection.request('POST', '/', body)
-        except Exception:
-            log.error('Hooks calling Connection failed on %s', connection.__dict__)
-            raise
-        response = connection.getresponse()
-
-        response_data = response.read()
+        connection = http.client.HTTPConnection(self.hooks_uri)
+        # binary msgpack body
+        headers, body = self._serialize(method, extras)
+        log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
 
         try:
-            return json.loads(response_data)
-        except Exception:
-            log.exception('Failed to decode hook response json data. '
-                          'response_code:%s, raw_data:%s',
-                          response.status, response_data)
-            raise
+            try:
+                connection.request('POST', '/', body, headers)
+            except Exception as error:
+                log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
+                raise
 
-    def _serialize(self, hook_name, extras):
+            response = connection.getresponse()
+            try:
+                return msgpack.load(response)
+            except Exception:
+                response_data = response.read()
+                log.exception('Failed to decode hook response json data. '
+                              'response_code:%s, raw_data:%s',
+                              response.status, response_data)
+                raise
+        finally:
+            connection.close()
+
+    @classmethod
+    def _serialize(cls, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
-        return json.dumps(data)
+        headers = {
+            "rc-hooks-protocol": cls.proto,
+            "Connection": "keep-alive"
+        }
+        return headers, msgpack.packb(data)
 
 
 class HooksDummyClient(object):
@@ -98,7 +115,7 b' class HgMessageWriter(RemoteMessageWrite'
     def __init__(self, ui):
         self.ui = ui
 
-    def write(self, message):
+    def write(self, message: str):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
@@ -112,8 +129,8 b' class GitMessageWriter(RemoteMessageWrit'
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
-    def write(self, message):
-        self.stdout.write(message.encode('utf-8'))
+    def write(self, message: str):
+        self.stdout.write(message)
 
 
 class SvnMessageWriter(RemoteMessageWriter):
@@ -130,6 +147,7 b' class SvnMessageWriter(RemoteMessageWrit'
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
+    log.debug('Handling hook-call exception: %s', exception_class)
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
@@ -141,13 +159,15 b' def _handle_exception(result):'
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
-        raise Exception('Got remote exception "%s" with args "%s"' %
-                        (exception_class, result['exception_args']))
+        raise Exception(
+            f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
+        )
 
 
 def _get_hooks_client(extras):
     hooks_uri = extras.get('hooks_uri')
     is_shadow_repo = extras.get('is_shadow_repo')
+
     if hooks_uri:
         return HooksHttpClient(extras['hooks_uri'])
     elif is_shadow_repo:
@@ -161,7 +181,6 b' def _call_hook(hook_name, extras, writer'
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
-
     _handle_exception(result)
     writer.write(result['output'])
 
@@ -169,7 +188,7 b' def _call_hook(hook_name, extras, writer'
 
 
 def _extras_from_ui(ui):
-    hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
+    hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
@@ -192,8 +211,8 b' def _rev_range_hash(repo, node, check_he'
     for rev in range(start, end):
         revs.append(rev)
         ctx = get_ctx(repo, rev)
-        commit_id = mercurial.node.hex(ctx.node())
-        branch = ctx.branch()
+        commit_id = ascii_str(mercurial.node.hex(ctx.node()))
+        branch = safe_str(ctx.branch())
         commits.append((commit_id, branch))
 
     parent_heads = []
@@ -217,9 +236,9 b' def _check_heads(repo, start, end, commi'
     for p in parents:
         branch = get_ctx(repo, p).branch()
         # The heads descending from that parent, on the same branch
-        parent_heads = set([p])
-        reachable = set([p])
-        for x in xrange(p + 1, end):
+        parent_heads = {p}
+        reachable = {p}
+        for x in range(p + 1, end):
             if get_ctx(repo, x).branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
@@ -295,14 +314,16 b' def pre_push(ui, repo, node=None, **kwar'
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
-    if node and kwargs.get('hooktype') == 'pretxnchangegroup':
+    hook_type: str = safe_str(kwargs.get('hooktype'))
+
+    if node and hook_type == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
-            old_rev = kwargs.get('node_last') or commits[0]
+            old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
@@ -319,10 +340,10 b' def pre_push(ui, repo, node=None, **kwar'
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
-                new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
+                new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
                 repo_path=repo_path)
 
-    extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
+    extras['hook_type'] = hook_type or 'pre_push'
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
@@ -363,6 +384,7 b' def post_push(ui, repo, node, **kwargs):'
     branches = []
     bookmarks = []
     tags = []
+    hook_type: str = safe_str(kwargs.get('hooktype'))
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
@@ -370,11 +392,12 b' def post_push(ui, repo, node, **kwargs):'
         if branch not in branches:
             branches.append(branch)
 
-    if hasattr(ui, '_rc_pushkey_branches'):
-        bookmarks = ui._rc_pushkey_branches
+    if hasattr(ui, '_rc_pushkey_bookmarks'):
+        bookmarks = ui._rc_pushkey_bookmarks
 
-    extras['hook_type'] = kwargs.get('hooktype', 'post_push')
+    extras['hook_type'] = hook_type or 'post_push'
     extras['commit_ids'] = commit_ids
+
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
@@ -395,9 +418,10 b' def post_push_ssh(ui, repo, node, **kwar'
 
 def key_push(ui, repo, **kwargs):
     from vcsserver.hgcompat import get_ctx
-    if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
+
+    if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
-        ui._rc_pushkey_branches = get_ctx(repo, kwargs['key']).bookmarks()
+        ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
     return
 
 
@@ -426,10 +450,13 b' def handle_git_post_receive(unused_repo_'
     pass
 
 
-HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
[e-mail]
+class HookResponse:
+    status: int
+    output: str
 
 
-def git_pre_pull(extras):
+def git_pre_pull(extras) -> HookResponse:
     """
     Pre pull hook.
 
@@ -439,20 +466,23 b' def git_pre_pull(extras):'
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
+
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
-    stdout = io.BytesIO()
+    stdout = io.StringIO()
     try:
-        status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
+        status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
+
     except Exception as error:
-        status = 128
-        stdout.write('ERROR: %s\n' % str(error))
+        log.exception('Failed to call pre_pull hook')
+        status_code = 128
+        stdout.write(f'ERROR: {error}\n')
 
-    return HookResponse(status, stdout.getvalue())
+    return HookResponse(status_code, stdout.getvalue())
 
 
-def git_post_pull(extras):
+def git_post_pull(extras) -> HookResponse:
     """
     Post pull hook.
 
@@ -465,12 +495,12 b' def git_post_pull(extras):'
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
-    stdout = io.BytesIO()
+    stdout = io.StringIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
-        stdout.write('ERROR: %s\n' % error)
+        stdout.write(f'ERROR: {error}\n')
 
     return HookResponse(status, stdout.getvalue())
 
@@ -495,15 +525,11 b' def _parse_git_ref_lines(revision_lines)'
     return rev_data
 
 
-def git_pre_receive(unused_repo_path, revision_lines, env):
+def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
     """
     Pre push hook.
 
-    :param extras: dictionary containing the keys defined in simplevcs
-    :type extras: dict
-
     :return: status code of the hook. 0 for success.
-    :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
@@ -527,7 +553,7 b' def git_pre_receive(unused_repo_path, re'
         if type_ == 'heads' and not (new_branch or delete_branch):
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
-            cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, '^{}'.format(new_rev)]
+            cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}']
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push was used
@@ -536,18 +562,18 b' def git_pre_receive(unused_repo_path, re'
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
-    return _call_hook('pre_push', extras, GitMessageWriter())
+
+    stdout = sys.stdout
+    status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
+
+    return status_code
 
 
-def git_post_receive(unused_repo_path, revision_lines, env):
+def git_post_receive(unused_repo_path, revision_lines, env) -> int:
     """
     Post push hook.
 
-    :param extras: dictionary containing the keys defined in simplevcs
-    :type extras: dict
-
     :return: status code of the hook. 0 for success.
-    :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
@@ -567,26 +593,28 b' def git_post_receive(unused_repo_path, r'
         type_ = push_ref['type']
 
         if type_ == 'heads':
+            # starting new branch case
             if push_ref['old_rev'] == empty_commit_id:
-                # starting new branch case
-                if push_ref['name'] not in branches:
-                    branches.append(push_ref['name'])
+                push_ref_name = push_ref['name']
+
+                if push_ref_name not in branches:
+                    branches.append(push_ref_name)
 
-                # Fix up head revision if needed
-                cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
-                try:
-                    subprocessio.run_command(cmd, env=os.environ.copy())
-                except Exception:
-                    cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"',
-                           '"refs/heads/%s"' % push_ref['name']]
-                    print("Setting default branch to %s" % push_ref['name'])
-                    subprocessio.run_command(cmd, env=os.environ.copy())
+                need_head_set = ''
+                with Repository(os.getcwd()) as repo:
+                    try:
+                        repo.head
+                    except pygit2.GitError:
+                        need_head_set = f'refs/heads/{push_ref_name}'
 
-                cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
-                       '--format=%(refname)', 'refs/heads/*']
+                    if need_head_set:
+                        repo.set_head(need_head_set)
+                        print(f"Setting default branch to {push_ref_name}")
+
+                cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
-                heads = stdout
+                heads = safe_str(stdout)
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
@@ -595,9 +623,10 b' def git_post_receive(unused_repo_path, r'
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
-                git_revs.extend(stdout.splitlines())
+                git_revs.extend(list(map(ascii_str, stdout.splitlines())))
+
+            # delete branch case
             elif push_ref['new_rev'] == empty_commit_id:
-                # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
@@ -608,7 +637,25 b' def git_post_receive(unused_repo_path, r'
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
-                git_revs.extend(stdout.splitlines())
+                # we get bytes from stdout, we need str to be consistent
+                log_revs = list(map(ascii_str, stdout.splitlines()))
+                git_revs.extend(log_revs)
+
+                # Pure pygit2 impl. but still 2-3x slower :/
+                # results = []
+                #
+                # with Repository(os.getcwd()) as repo:
+                #     repo_new_rev = repo[push_ref['new_rev']]
+                #     repo_old_rev = repo[push_ref['old_rev']]
+                #     walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
+                #
+                #     for commit in walker:
+                #         if commit.id == repo_old_rev.id:
+                #             break
+                #         results.append(commit.id.hex)
+                #     # reverse the order, can't use GIT_SORT_REVERSE
+                #     log_revs = results[::-1]
+
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
@@ -622,13 +669,16 b' def git_post_receive(unused_repo_path, r'
         'tags': tags,
     }
 
+    stdout = sys.stdout
+
     if 'repo_size' in extras['hooks']:
         try:
-            _call_hook('repo_size', extras, GitMessageWriter())
-        except:
+            _call_hook('repo_size', extras, GitMessageWriter(stdout))
+        except Exception:
             pass
 
-    return _call_hook('post_push', extras, GitMessageWriter())
+    status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
+    return status_code
 
 
 def _get_extras_from_txn_id(path, txn_id):

vcsserver/http_main.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,9 +15,9 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import io
 import os
 import sys
-import base64
 import locale
 import logging
 import uuid
@@ -28,17 +28,20 b' import tempfile'
 import psutil
 
 from itertools import chain
-from cStringIO import StringIO
 
-import simplejson as json
 import msgpack
+import configparser
+
 from pyramid.config import Configurator
 from pyramid.wsgi import wsgiapp
-from pyramid.compat import configparser
 from pyramid.response import Response
+
+from vcsserver.base import BytesEnvelope, BinaryEnvelope
+from vcsserver.lib.rc_json import json
 from vcsserver.config.settings_maker import SettingsMaker
-from vcsserver.utils import safe_int
+from vcsserver.str_utils import safe_int
 from vcsserver.lib.statsd_client import StatsdClient
+from vcsserver.tweens.request_wrapper import get_call_context, get_headers_call_context
 
 log = logging.getLogger(__name__)
 
@@ -62,23 +65,39 b' from vcsserver.exceptions import HTTPRep'
 from vcsserver.lib.exc_tracking import store_exception
 from vcsserver.server import VcsServer
 
+strict_vcs = True
+
+git_import_err = None
 try:
-    from vcsserver.git import GitFactory, GitRemote
-except ImportError:
+    from vcsserver.remote.git import GitFactory, GitRemote
+except ImportError as e:
     GitFactory = None
     GitRemote = None
+    git_import_err = e
+    if strict_vcs:
+        raise
 
+
+hg_import_err = None
 try:
-    from vcsserver.hg import MercurialFactory, HgRemote
-except ImportError:
+    from vcsserver.remote.hg import MercurialFactory, HgRemote
+except ImportError as e:
     MercurialFactory = None
     HgRemote = None
+    hg_import_err = e
+    if strict_vcs:
+        raise
 
+
+svn_import_err = None
 try:
-    from vcsserver.svn import SubversionFactory, SvnRemote
-except ImportError:
+    from vcsserver.remote.svn import SubversionFactory, SvnRemote
+except ImportError as e:
     SubversionFactory = None
     SvnRemote = None
+    svn_import_err = e
+    if strict_vcs:
+        raise
 
 
 def _is_request_chunked(environ):
@@ -106,13 +125,13 b' class VCS(object):'
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
-            log.info("Git client import failed")
+            log.error("Git client import failed: %s", git_import_err)
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
-            log.info("Mercurial client import failed")
+            log.error("Mercurial client import failed: %s", hg_import_err)
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
@@ -121,7 +140,7 b' class VCS(object):'
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
-            log.warning("Subversion client import failed")
+            log.error("Subversion client import failed: %s", svn_import_err)
 
         self._vcsserver = VcsServer()
 
@@ -129,8 +148,7 b' class VCS(object):'
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
-            log.info(
-                'Configuring locale subsystem based on environment variables')
+            log.info('Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
@@ -142,8 +160,7 b' class VCS(object):'
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
-            log.exception(
-                'Cannot set locale, not configuring the locale system')
+            log.exception('Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
@@ -189,7 +206,7 b' class VCSViewPredicate(object):'
         self.remotes = val
 
     def text(self):
-        return 'vcs view method = %s' % (self.remotes.keys(),)
+        return f'vcs view method = {list(self.remotes.keys())}'
 
     phash = text
 
@@ -213,9 +230,11 b' class HTTPApplication(object):'
         self.config = Configurator(settings=settings)
         # Init our statsd at very start
         self.config.registry.statsd = StatsdClient.statsd
+        self.config.registry.vcs_call_context = {}
 
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
+        self.config.include('vcsserver.lib.rc_cache.archive_cache')
 
         settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale_conf=settings_locale, cache_config=settings)
@@ -302,6 +321,7 b' class HTTPApplication(object):'
     def _vcs_view_params(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
+
         method = payload.get('method')
         params = payload['params']
         wire = params.get('wire')
@@ -309,6 +329,11 b' class HTTPApplication(object):'
         kwargs = params.get('kwargs')
         context_uid = None
 
+        request.registry.vcs_call_context = {
+            'method': method,
+            'repo_name': payload.get('_repo_name'),
+        }
+
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
@@ -319,22 +344,33 b' class HTTPApplication(object):'
 
         # NOTE(marcink): trading complexity for slight performance
         if log.isEnabledFor(logging.DEBUG):
-            no_args_methods = [
-
-            ]
-            if method in no_args_methods:
+            # also we SKIP printing out any of those methods args since they maybe excessive
+            just_args_methods = {
+                'commitctx': ('content', 'removed', 'updated'),
+                'commit': ('content', 'removed', 'updated')
+            }
+            if method in just_args_methods:
+                skip_args = just_args_methods[method]
                 call_args = ''
+                call_kwargs = {}
+                for k in kwargs:
+                    if k in skip_args:
+                        # replace our skip key with dummy
+                        call_kwargs[k] = f'RemovedParam({k})'
+                    else:
+                        call_kwargs[k] = kwargs[k]
             else:
                 call_args = args[1:]
+                call_kwargs = kwargs
 
             log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
-                      method, call_args, kwargs, context_uid, repo_state_uid)
+                      method, call_args, call_kwargs, context_uid, repo_state_uid)
 
         statsd = request.registry.statsd
         if statsd:
             statsd.incr(
                 'vcsserver_method_total', tags=[
-                    "method:{}".format(method),
+                    f"method:{method}",
                 ])
         return payload, remote, method, args, kwargs
 
@@ -384,7 +420,7 b' class HTTPApplication(object):'
             resp = {
                 'id': payload_id,
                 'error': {
-                    'message': e.message,
+                    'message': str(e),
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'org_exc_tb': org_exc_tb,
@@ -401,7 +437,7 b' class HTTPApplication(object):'
                 'id': payload_id,
                 'result': resp
             }
-
+        log.debug('Serving data for method %s', method)
         return resp
 
     def vcs_stream_view(self, request):
@@ -416,7 +452,7 b' class HTTPApplication(object):'
             raise
 
         def get_chunked_data(method_resp):
-            stream = StringIO(method_resp)
+            stream = io.BytesIO(method_resp)
             while 1:
                 chunk = stream.read(chunk_size)
                 if not chunk:
@@ -453,7 +489,7 b' class HTTPApplication(object):'
         except Exception:
             log.exception('Failed to read .ini file for display')
 
-        environ = os.environ.items()
+        environ = list(os.environ.items())
 
         resp = {
             'id': payload.get('id'),
@@ -468,14 +504,28 b' class HTTPApplication(object):'
         return resp
 
     def _msgpack_renderer_factory(self, info):
+
         def _render(value, system):
+            bin_type = False
+            res = value.get('result')
+            if isinstance(res, BytesEnvelope):
+                log.debug('Result is wrapped in BytesEnvelope type')
+                bin_type = True
+            elif isinstance(res, BinaryEnvelope):
+                log.debug('Result is wrapped in BinaryEnvelope type')
+                value['result'] = res.val
+                bin_type = True
+
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
-            return msgpack.packb(value)
+                    if bin_type:
+                        response.content_type = 'application/x-msgpack-bin'
+
+            return msgpack.packb(value, use_bin_type=bin_type)
         return _render
 
     def set_env_from_config(self, environ, config):
@@ -528,16 +578,17 b' class HTTPApplication(object):'
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
-                repo_path = environ['HTTP_X_RC_REPO_PATH']
-                repo_name = environ['HTTP_X_RC_REPO_NAME']
-                packed_config = base64.b64decode(
-                    environ['HTTP_X_RC_REPO_CONFIG'])
-                config = msgpack.unpackb(packed_config)
+                call_context = get_headers_call_context(environ)
+
+                repo_path = call_context['repo_path']
+                repo_name = call_context['repo_name']
+                config = call_context['repo_config']
+
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
-                environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
+                environ['PATH_INFO'] = call_context['path_info']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
@@ -557,13 +608,14 b' class HTTPApplication(object):'
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
-                repo_path = environ['HTTP_X_RC_REPO_PATH']
-                repo_name = environ['HTTP_X_RC_REPO_NAME']
-                packed_config = base64.b64decode(
-                    environ['HTTP_X_RC_REPO_CONFIG'])
-                config = msgpack.unpackb(packed_config)
+
+                call_context = get_headers_call_context(environ)
 
-                environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
+                repo_path = call_context['repo_path']
+                repo_name = call_context['repo_name']
+                config = call_context['repo_config']
+
+                environ['PATH_INFO'] = call_context['path_info']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
@@ -599,15 +651,18 b' class HTTPApplication(object):'
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
+
         if _vcs_kind == 'repo_locked':
-            # Get custom repo-locked status code if present.
-            status_code = request.headers.get('X-RC-Locked-Status-Code')
+            headers_call_context = get_headers_call_context(request.environ)
+            status_code = safe_int(headers_call_context['locked_status_code'])
+
             return HTTPRepoLocked(
-                title=exception.message, status_code=status_code)
+                title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
-            return HTTPRepoBranchProtected(title=exception.message)
+            return HTTPRepoBranchProtected(
+                title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
@@ -623,9 +678,9 b' class HTTPApplication(object):'
 
         statsd = request.registry.statsd
         if statsd:
-            exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
+            exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
             statsd.incr('vcsserver_exception_total',
-                        tags=["type:{}".format(exc_type)])
+                        tags=[f"type:{exc_type}"])
         raise exception
 
 
@@ -716,8 +771,7 b' def main(global_config, **settings):'
 
     pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
     total_time = time.time() - start_time
-    log.info('Pyramid app `%s` created and configured in %.2fs',
-             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
+    log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app
 
 

vcsserver/lib/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/lib/_vendor/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/lib/_vendor/jsonlogger/__init__.py

@@ -84,7 +84,7 b' class JsonEncoder(json.JSONEncoder):'
             return str(obj)
 
         try:
-            return super(JsonEncoder, self).default(obj)
+            return super().default(obj)
 
         except TypeError:
             try:
@@ -132,7 +132,7 b' class JsonFormatter(ExceptionAwareFormat'
         self.json_ensure_ascii = kwargs.pop("json_ensure_ascii", True)
         self.prefix = kwargs.pop("prefix", "")
         reserved_attrs = kwargs.pop("reserved_attrs", RESERVED_ATTRS)
-        self.reserved_attrs = dict(zip(reserved_attrs, reserved_attrs))
+        self.reserved_attrs = dict(list(zip(reserved_attrs, reserved_attrs)))
         self.timestamp = kwargs.pop("timestamp", True)
 
         # super(JsonFormatter, self).__init__(*args, **kwargs)
@@ -141,8 +141,8 b' class JsonFormatter(ExceptionAwareFormat'
             self.json_encoder = JsonEncoder
 
         self._required_fields = self.parse()
-        self._skip_fields = dict(zip(self._required_fields,
-                                     self._required_fields))
+        self._skip_fields = dict(list(zip(self._required_fields,
+                                      self._required_fields)))
         self._skip_fields.update(self.reserved_attrs)
 
     def _str_to_fn(self, fn_as_str):
@@ -200,7 +200,7 b' class JsonFormatter(ExceptionAwareFormat'
 
     def serialize_log_record(self, log_record):
         """Returns the final representation of the log record."""
-        return "%s%s" % (self.prefix, self.jsonify_log_record(log_record))
+        return "{}{}".format(self.prefix, self.jsonify_log_record(log_record))
 
     def format(self, record):
         """Formats a log record and serializes to json"""

vcsserver/lib/_vendor/redis_lock/__init__.py

@@ -23,14 +23,8 b' loggers = {'
     ]
 }
 
-PY3 = sys.version_info[0] == 3
-
-if PY3:
-    text_type = str
-    binary_type = bytes
-else:
-    text_type = unicode  # noqa
-    binary_type = str
+text_type = str
+binary_type = bytes
 
 
 # Check if the id match. If not, return an error code.
@@ -225,7 +219,7 b' class Lock(object):'
 
         if self._held:
             owner_id = self.get_owner_id()
-            raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
+            raise AlreadyAcquired(f"Already acquired from this Lock instance. Lock id: {owner_id}")
 
         if not blocking and timeout is not None:
             raise TimeoutNotUsable("Timeout cannot be used if blocking=False")

vcsserver/lib/_vendor/statsd/__init__.py

@@ -1,5 +1,3 b''
-from __future__ import absolute_import, division, unicode_literals
-
 import logging
 
 from .stream import TCPStatsClient, UnixSocketStatsClient  # noqa

vcsserver/lib/_vendor/statsd/base.py

@@ -1,5 +1,3 b''
-from __future__ import absolute_import, division, unicode_literals
-
 import re
 import random
 from collections import deque
@@ -49,7 +47,7 b' class StatsClientBase(object):'
         statsd = StatsdClient.statsd
         with statsd.timer('bucket_name', auto_send=True) as tmr:
             # This block will be timed.
-            for i in xrange(0, 100000):
+            for i in range(0, 100000):
                 i ** 2
         # you can access time here...
         elapsed_ms = tmr.ms

vcsserver/lib/_vendor/statsd/stream.py

@@ -1,5 +1,3 b''
-from __future__ import absolute_import, division, unicode_literals
-
 import socket
 
 from .base import StatsClientBase, PipelineBase

vcsserver/lib/_vendor/statsd/timer.py

@@ -1,14 +1,5 b''
-from __future__ import absolute_import, division, unicode_literals
-
 import functools
-
-# Use timer that's not susceptible to time of day adjustments.
-try:
-    # perf_counter is only present on Py3.3+
-    from time import perf_counter as time_now
-except ImportError:
-    # fall back to using time
-    from time import time as time_now
+from time import perf_counter as time_now
 
 
 def safe_wraps(wrapper, *args, **kwargs):

vcsserver/lib/_vendor/statsd/udp.py

@@ -1,5 +1,3 b''
-from __future__ import absolute_import, division, unicode_literals
-
 import socket
 
 from .base import StatsClientBase, PipelineBase
@@ -8,7 +6,7 b' from .base import StatsClientBase, Pipel'
 class Pipeline(PipelineBase):
 
     def __init__(self, client):
-        super(Pipeline, self).__init__(client)
+        super().__init__(client)
         self._maxudpsize = client._maxudpsize
 
     def _send(self):

vcsserver/lib/exc_tracking.py

@@ -1,7 +1,5 b''
-# -*- coding: utf-8 -*-
-
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -26,8 +24,6 b' import logging'
 import traceback
 import tempfile
 
-from pyramid import compat
-
 log = logging.getLogger(__name__)
 
 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
@@ -77,7 +73,8 b' def _store_exception(exc_id, exc_info, p'
     detailed_tb = getattr(exc_value, '_org_exc_tb', None)
 
     if detailed_tb:
-        if isinstance(detailed_tb, compat.string_types):
+        remote_tb = detailed_tb
+        if isinstance(detailed_tb, str):
             remote_tb = [detailed_tb]
 
         tb += (
@@ -127,10 +124,10 b' def store_exception(exc_id, exc_info, pr'
 def _find_exc_file(exc_id, prefix=global_prefix):
     exc_store_path = get_exc_store()
     if prefix:
-        exc_id = '{}_{}'.format(exc_id, prefix)
+        exc_id = f'{exc_id}_{prefix}'
     else:
         # search without a prefix
-        exc_id = '{}'.format(exc_id)
+        exc_id = f'{exc_id}'
 
     # we need to search the store for such start pattern as above
     for fname in os.listdir(exc_store_path):

vcsserver/lib/memory_lru_dict.py

@@ -1,7 +1,5 b''
-# -*- coding: utf-8 -*-
-
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,7 +20,7 b' import logging'
 
 from repoze.lru import LRUCache
 
-from vcsserver.utils import safe_str
+from vcsserver.str_utils import safe_str
 
 log = logging.getLogger(__name__)
 
@@ -45,7 +43,7 b' class LRUDict(LRUCache):'
         del self.data[key]
 
     def keys(self):
-        return self.data.keys()
+        return list(self.data.keys())
 
 
 class LRUDictDebug(LRUDict):
@@ -53,11 +51,11 b' class LRUDictDebug(LRUDict):'
     Wrapper to provide some debug options
     """
     def _report_keys(self):
-        elems_cnt = '%s/%s' % (len(self.keys()), self.size)
+        elems_cnt = f'{len(list(self.keys()))}/{self.size}'
         # trick for pformat print it more nicely
         fmt = '\n'
         for cnt, elem in enumerate(self.keys()):
-            fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
+            fmt += f'{cnt+1} - {safe_str(elem)}\n'
         log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
 
     def __getitem__(self, key):

vcsserver/lib/rc_cache/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,31 +16,57 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
+import threading
+
 from dogpile.cache import register_backend
 
+from . import region_meta
+from .utils import (
+    backend_key_generator,
+    clear_cache_namespace,
+    get_default_cache_settings,
+    get_or_create_region,
+    make_region,
+    str2bool,
+)
+
+module_name = 'vcsserver'
+
 register_backend(
-    "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
+    "dogpile.cache.rc.memory_lru", f"{module_name}.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
 register_backend(
-    "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
+    "dogpile.cache.rc.file_namespace", f"{module_name}.lib.rc_cache.backends",
     "FileNamespaceBackend")
 
 register_backend(
-    "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
+    "dogpile.cache.rc.redis", f"{module_name}.lib.rc_cache.backends",
     "RedisPickleBackend")
 
 register_backend(
-    "dogpile.cache.rc.redis_msgpack", "vcsserver.lib.rc_cache.backends",
+    "dogpile.cache.rc.redis_msgpack", f"{module_name}.lib.rc_cache.backends",
     "RedisMsgPackBackend")
 
 
 log = logging.getLogger(__name__)
 
-from . import region_meta
-from .utils import (
-    get_default_cache_settings, backend_key_generator, get_or_create_region,
-    clear_cache_namespace, make_region)
+
+CLEAR_DELETE = 'delete'
+CLEAR_INVALIDATE = 'invalidate'
+
+
+def async_creation_runner(cache, somekey, creator, mutex):
+
+    def runner():
+        try:
+            value = creator()
+            cache.set(somekey, value)
+        finally:
+            mutex.release()
+
+    thread = threading.Thread(target=runner)
+    thread.start()
 
 
 def configure_dogpile_cache(settings):
@@ -62,13 +88,20 b' def configure_dogpile_cache(settings):'
 
         new_region = make_region(
             name=namespace_name,
-            function_key_generator=None
+            function_key_generator=None,
+            async_creation_runner=None
         )
 
-        new_region.configure_from_config(settings, 'rc_cache.{}.'.format(namespace_name))
+        new_region.configure_from_config(settings, f'rc_cache.{namespace_name}.')
         new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
+
+        async_creator = str2bool(settings.pop(f'rc_cache.{namespace_name}.async_creator', 'false'))
+        if async_creator:
+            log.debug('configuring region %s with async creator', new_region)
+            new_region.async_creation_runner = async_creation_runner
+
         if log.isEnabledFor(logging.DEBUG):
-            region_args = dict(backend=new_region.actual_backend.__class__,
+            region_args = dict(backend=new_region.actual_backend,
                                region_invalidator=new_region.region_invalidator.__class__)
             log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
 

vcsserver/lib/rc_cache/backends.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,25 +15,31 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
-import time
 import errno
+import fcntl
+import functools
 import logging
+import os
+import pickle
+#import time
 
+#import gevent
 import msgpack
 import redis
 
-from dogpile.cache.api import CachedValue
-from dogpile.cache.backends import memory as memory_backend
+flock_org = fcntl.flock
+from typing import Union
+
+from dogpile.cache.api import Deserializer, Serializer
 from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import redis as redis_backend
-from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
+from dogpile.cache.backends.file import FileLock
 from dogpile.cache.util import memoized_property
 
-from pyramid.settings import asbool
-
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
-from vcsserver.utils import safe_str, safe_unicode
-
+from vcsserver.str_utils import safe_bytes, safe_str
+from vcsserver.type_utils import str2bool
 
 _default_max_size = 1024
 
@@ -45,14 +51,20 b' class LRUMemoryBackend(memory_backend.Me'
     pickle_values = False
 
     def __init__(self, arguments):
-        max_size = arguments.pop('max_size', _default_max_size)
+        self.max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
-        arguments['cache_dict'] = LRUDictClass(max_size)
-        super(LRUMemoryBackend, self).__init__(arguments)
+        arguments['cache_dict'] = LRUDictClass(self.max_size)
+        super().__init__(arguments)
+
+    def __repr__(self):
+        return f'{self.__class__}(maxsize=`{self.max_size}`)'
+
+    def __str__(self):
+        return self.__repr__()
 
     def delete(self, key):
         try:
@@ -66,55 +78,22 b' class LRUMemoryBackend(memory_backend.Me'
             self.delete(key)
 
 
-class PickleSerializer(object):
-
-    def _dumps(self, value, safe=False):
-        try:
-            return compat.pickle.dumps(value)
-        except Exception:
-            if safe:
-                return NO_VALUE
-            else:
-                raise
-
-    def _loads(self, value, safe=True):
-        try:
-            return compat.pickle.loads(value)
-        except Exception:
-            if safe:
-                return NO_VALUE
-            else:
-                raise
+class PickleSerializer:
+    serializer: None | Serializer = staticmethod(  # type: ignore
+        functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
+    )
+    deserializer: None | Deserializer = staticmethod(  # type: ignore
+        functools.partial(pickle.loads)
+    )
 
 
 class MsgPackSerializer(object):
-
-    def _dumps(self, value, safe=False):
-        try:
-            return msgpack.packb(value)
-        except Exception:
-            if safe:
-                return NO_VALUE
-            else:
-                raise
-
-    def _loads(self, value, safe=True):
-        """
-        pickle maintained the `CachedValue` wrapper of the tuple
-        msgpack does not, so it must be added back in.
-       """
-        try:
-            value = msgpack.unpackb(value, use_list=False)
-            return CachedValue(*value)
-        except Exception:
-            if safe:
-                return NO_VALUE
-            else:
-                raise
-
-
-import fcntl
-flock_org = fcntl.flock
+    serializer: None | Serializer = staticmethod(  # type: ignore
+        msgpack.packb
+    )
+    deserializer: None | Deserializer = staticmethod(  # type: ignore
+        functools.partial(msgpack.unpackb, use_list=False)
+    )
 
 
 class CustomLockFactory(FileLock):
@@ -129,30 +108,40 b' class FileNamespaceBackend(PickleSeriali'
         arguments['lock_factory'] = CustomLockFactory
         db_file = arguments.get('filename')
 
-        log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
+        log.debug('initialing cache-backend=%s db in %s', self.__class__.__name__, db_file)
+        db_file_dir = os.path.dirname(db_file)
+        if not os.path.isdir(db_file_dir):
+            os.makedirs(db_file_dir)
+
         try:
-            super(FileNamespaceBackend, self).__init__(arguments)
+            super().__init__(arguments)
         except Exception:
             log.exception('Failed to initialize db at: %s', db_file)
             raise
 
     def __repr__(self):
-        return '{} `{}`'.format(self.__class__, self.filename)
+        return f'{self.__class__}(file=`{self.filename}`)'
+
+    def __str__(self):
+        return self.__repr__()
 
-    def list_keys(self, prefix=''):
-        prefix = '{}:{}'.format(self.key_prefix, prefix)
+    def _get_keys_pattern(self, prefix: bytes = b''):
+        return b'%b:%b' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
 
-        def cond(v):
+    def list_keys(self, prefix: bytes = b''):
+        prefix = self._get_keys_pattern(prefix)
+
+        def cond(dbm_key: bytes):
             if not prefix:
                 return True
 
-            if v.startswith(prefix):
+            if dbm_key.startswith(prefix):
                 return True
             return False
 
         with self._dbm_file(True) as dbm:
             try:
-                return filter(cond, dbm.keys())
+                return list(filter(cond, dbm.keys()))
             except Exception:
                 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
                 raise
@@ -160,49 +149,27 b' class FileNamespaceBackend(PickleSeriali'
     def get_store(self):
         return self.filename
 
-    def _dbm_get(self, key):
-        with self._dbm_file(False) as dbm:
-            if hasattr(dbm, 'get'):
-                value = dbm.get(key, NO_VALUE)
-            else:
-                # gdbm objects lack a .get method
-                try:
-                    value = dbm[key]
-                except KeyError:
-                    value = NO_VALUE
-            if value is not NO_VALUE:
-                value = self._loads(value)
-            return value
-
-    def get(self, key):
-        try:
-            return self._dbm_get(key)
-        except Exception:
-            log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
-            raise
-
-    def set(self, key, value):
-        with self._dbm_file(True) as dbm:
-            dbm[key] = self._dumps(value)
-
-    def set_multi(self, mapping):
-        with self._dbm_file(True) as dbm:
-            for key, value in mapping.items():
-                dbm[key] = self._dumps(value)
-
 
 class BaseRedisBackend(redis_backend.RedisBackend):
     key_prefix = ''
 
     def __init__(self, arguments):
-        super(BaseRedisBackend, self).__init__(arguments)
+        self.db_conn = arguments.get('host', '') or arguments.get('url', '') or 'redis-host'
+        super().__init__(arguments)
+
         self._lock_timeout = self.lock_timeout
-        self._lock_auto_renewal = asbool(arguments.pop("lock_auto_renewal", True))
+        self._lock_auto_renewal = str2bool(arguments.pop("lock_auto_renewal", True))
 
         if self._lock_auto_renewal and not self._lock_timeout:
             # set default timeout for auto_renewal
             self._lock_timeout = 30
 
+    def __repr__(self):
+        return f'{self.__class__}(conn=`{self.db_conn}`)'
+
+    def __str__(self):
+        return self.__repr__()
+
     def _create_client(self):
         args = {}
 
@@ -216,58 +183,29 b' class BaseRedisBackend(redis_backend.Red'
             )
 
         connection_pool = redis.ConnectionPool(**args)
-
-        return redis.StrictRedis(connection_pool=connection_pool)
+        self.writer_client = redis.StrictRedis(
+            connection_pool=connection_pool
+        )
+        self.reader_client = self.writer_client
 
-    def list_keys(self, prefix=''):
-        prefix = '{}:{}*'.format(self.key_prefix, prefix)
-        return self.client.keys(prefix)
+    def _get_keys_pattern(self, prefix: bytes = b''):
+        return b'%b:%b*' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
+
+    def list_keys(self, prefix: bytes = b''):
+        prefix = self._get_keys_pattern(prefix)
+        return self.reader_client.keys(prefix)
 
     def get_store(self):
-        return self.client.connection_pool
-
-    def get(self, key):
-        value = self.client.get(key)
-        if value is None:
-            return NO_VALUE
-        return self._loads(value)
-
-    def get_multi(self, keys):
-        if not keys:
-            return []
-        values = self.client.mget(keys)
-        loads = self._loads
-        return [
-            loads(v) if v is not None else NO_VALUE
-            for v in values]
-
-    def set(self, key, value):
-        if self.redis_expiration_time:
-            self.client.setex(key, self.redis_expiration_time,
-                              self._dumps(value))
-        else:
-            self.client.set(key, self._dumps(value))
-
-    def set_multi(self, mapping):
-        dumps = self._dumps
-        mapping = dict(
-            (k, dumps(v))
-            for k, v in mapping.items()
-        )
-
-        if not self.redis_expiration_time:
-            self.client.mset(mapping)
-        else:
-            pipe = self.client.pipeline()
-            for key, value in mapping.items():
-                pipe.setex(key, self.redis_expiration_time, value)
-            pipe.execute()
+        return self.reader_client.connection_pool
 
     def get_mutex(self, key):
         if self.distributed_lock:
-            lock_key = u'_lock_{0}'.format(safe_unicode(key))
-            return get_mutex_lock(self.client, lock_key, self._lock_timeout,
-                                  auto_renewal=self._lock_auto_renewal)
+            lock_key = f'_lock_{safe_str(key)}'
+            return get_mutex_lock(
+                self.writer_client, lock_key,
+                self._lock_timeout,
+                auto_renewal=self._lock_auto_renewal
+            )
         else:
             return None
 
@@ -283,7 +221,7 b' class RedisMsgPackBackend(MsgPackSeriali'
 
 
 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
-    import redis_lock
+    from vcsserver.lib._vendor import redis_lock
 
     class _RedisLockWrapper(object):
         """LockWrapper for redis_lock"""
@@ -299,10 +237,10 b' def get_mutex_lock(client, lock_key, loc'
             )
 
         def __repr__(self):
-            return "{}:{}".format(self.__class__.__name__, lock_key)
+            return f"{self.__class__.__name__}:{lock_key}"
 
         def __str__(self):
-            return "{}:{}".format(self.__class__.__name__, lock_key)
+            return f"{self.__class__.__name__}:{lock_key}"
 
         def __init__(self):
             self.lock = self.get_lock()

vcsserver/lib/rc_cache/region_meta.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/lib/rc_cache/utils.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,115 +15,69 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
-import os
-import time
+import functools
 import logging
-import functools
+import os
+import threading
+import time
 
+import decorator
 from dogpile.cache import CacheRegion
-from dogpile.cache.util import compat
+
 
-from vcsserver.utils import safe_str, sha1
+from vcsserver.utils import sha1
+from vcsserver.str_utils import safe_bytes
+from vcsserver.type_utils import str2bool
 
-from vcsserver.lib.rc_cache import region_meta
+from . import region_meta
 
 log = logging.getLogger(__name__)
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
+    def __repr__(self):
+        return f'{self.__class__}(name={self.name})'
+
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
-            to_str=compat.string_type,
+            to_str=str,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
-        condition isn't meet. This works a bit different than should_cache_fn
+        condition isn't meet. This works a bit different from should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
-        expiration_time_is_callable = compat.callable(expiration_time)
+        expiration_time_is_callable = callable(expiration_time)
+        if not namespace:
+            namespace = getattr(self, '_default_namespace', None)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
-        # workaround for py2 and cython problems, this block should be removed
-        # once we've migrated to py3
-        if 'cython' == 'cython':
-            def decorator(fn):
-                if to_str is compat.string_type:
-                    # backwards compatible
-                    key_generator = function_key_generator(namespace, fn)
-                else:
-                    key_generator = function_key_generator(namespace, fn, to_str=to_str)
-
-                @functools.wraps(fn)
-                def decorate(*arg, **kw):
-                    key = key_generator(*arg, **kw)
-
-                    @functools.wraps(fn)
-                    def creator():
-                        return fn(*arg, **kw)
-
-                    if not condition:
-                        return creator()
-
-                    timeout = expiration_time() if expiration_time_is_callable \
-                        else expiration_time
-
-                    return self.get_or_create(key, creator, timeout, should_cache_fn)
-
-                def invalidate(*arg, **kw):
-                    key = key_generator(*arg, **kw)
-                    self.delete(key)
-
-                def set_(value, *arg, **kw):
-                    key = key_generator(*arg, **kw)
-                    self.set(key, value)
-
-                def get(*arg, **kw):
-                    key = key_generator(*arg, **kw)
-                    return self.get(key)
-
-                def refresh(*arg, **kw):
-                    key = key_generator(*arg, **kw)
-                    value = fn(*arg, **kw)
-                    self.set(key, value)
-                    return value
-
-                decorate.set = set_
-                decorate.invalidate = invalidate
-                decorate.refresh = refresh
-                decorate.get = get
-                decorate.original = fn
-                decorate.key_generator = key_generator
-                decorate.__wrapped__ = fn
-
-                return decorate
-            return decorator
-
-        def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
+        def get_or_create_for_user_func(func_key_generator, user_func, *arg, **kw):
 
             if not condition:
-                log.debug('Calling un-cached method:%s', user_func.func_name)
+                log.debug('Calling un-cached method:%s', user_func.__name__)
                 start = time.time()
                 result = user_func(*arg, **kw)
                 total = time.time() - start
-                log.debug('un-cached method:%s took %.4fs', user_func.func_name, total)
+                log.debug('un-cached method:%s took %.4fs', user_func.__name__, total)
                 return result
 
-            key = key_generator(*arg, **kw)
+            key = func_key_generator(*arg, **kw)
 
             timeout = expiration_time() if expiration_time_is_callable \
                 else expiration_time
 
-            log.debug('Calling cached method:`%s`', user_func.func_name)
+            log.debug('Calling cached method:`%s`', user_func.__name__)
             return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
 
         def cache_decorator(user_func):
-            if to_str is compat.string_type:
+            if to_str is str:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, user_func)
             else:
@@ -176,7 +130,7 b' def get_default_cache_settings(settings,'
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
-                if isinstance(val, compat.string_types):
+                if isinstance(val, str):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
@@ -186,7 +140,21 b' def compute_key_from_params(*args):'
     """
     Helper to compute key from given params to be used in cache manager
     """
-    return sha1("_".join(map(safe_str, args)))
+    return sha1(safe_bytes("_".join(map(str, args))))
+
+
+def custom_key_generator(backend, namespace, fn):
+    func_name = fn.__name__
+
+    def generate_key(*args):
+        backend_pref = getattr(backend, 'key_prefix', None) or 'backend_prefix'
+        namespace_pref = namespace or 'default_namespace'
+        arg_key = compute_key_from_params(*args)
+        final_key = f"{backend_pref}:{namespace_pref}:{func_name}_{arg_key}"
+
+        return final_key
+
+    return generate_key
 
 
 def backend_key_generator(backend):
@@ -194,49 +162,50 b' def backend_key_generator(backend):'
     Special wrapper that also sends over the backend to the key generator
     """
     def wrapper(namespace, fn):
-        return key_generator(backend, namespace, fn)
+        return custom_key_generator(backend, namespace, fn)
     return wrapper
 
 
-def key_generator(backend, namespace, fn):
-    fname = fn.__name__
+def get_or_create_region(region_name, region_namespace: str = None, use_async_runner=False):
+    from .backends import FileNamespaceBackend
+    from . import async_creation_runner
 
-    def generate_key(*args):
-        backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
-        namespace_pref = namespace or 'default_namespace'
-        arg_key = compute_key_from_params(*args)
-        final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
-
-        return final_key
-
-    return generate_key
-
-
-def get_or_create_region(region_name, region_namespace=None):
-    from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
     region_obj = region_meta.dogpile_cache_regions.get(region_name)
     if not region_obj:
-        raise EnvironmentError(
-            'Region `{}` not in configured: {}.'.format(
-                region_name, region_meta.dogpile_cache_regions.keys()))
+        reg_keys = list(region_meta.dogpile_cache_regions.keys())
+        raise OSError(f'Region `{region_name}` not in configured: {reg_keys}.')
+
+    region_uid_name = f'{region_name}:{region_namespace}'
 
-    region_uid_name = '{}:{}'.format(region_name, region_namespace)
     if isinstance(region_obj.actual_backend, FileNamespaceBackend):
+        if not region_namespace:
+            raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
+
         region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
         if region_exist:
             log.debug('Using already configured region: %s', region_namespace)
             return region_exist
-        cache_dir = region_meta.dogpile_config_defaults['cache_dir']
+
         expiration_time = region_obj.expiration_time
 
-        if not os.path.isdir(cache_dir):
-            os.makedirs(cache_dir)
+        cache_dir = region_meta.dogpile_config_defaults['cache_dir']
+        namespace_cache_dir = cache_dir
+
+        # we default the namespace_cache_dir to our default cache dir.
+        # however if this backend is configured with filename= param, we prioritize that
+        # so all caches within that particular region, even those namespaced end up in the same path
+        if region_obj.actual_backend.filename:
+            namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
+
+        if not os.path.isdir(namespace_cache_dir):
+            os.makedirs(namespace_cache_dir)
         new_region = make_region(
             name=region_uid_name,
             function_key_generator=backend_key_generator(region_obj.actual_backend)
         )
+
         namespace_filename = os.path.join(
-            cache_dir, "{}.cache.dbm".format(region_namespace))
+            namespace_cache_dir, f"{region_name}_{region_namespace}.cache_db")
         # special type that allows 1db per namespace
         new_region.configure(
             backend='dogpile.cache.rc.file_namespace',
@@ -248,16 +217,31 b' def get_or_create_region(region_name, re'
         log.debug('configuring new region: %s', region_uid_name)
         region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
 
+    region_obj._default_namespace = region_namespace
+    if use_async_runner:
+        region_obj.async_creation_runner = async_creation_runner
     return region_obj
 
 
-def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
-    region = get_or_create_region(cache_region, cache_namespace_uid)
-    cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
-    num_delete_keys = len(cache_keys)
-    if invalidate:
-        region.invalidate(hard=False)
-    else:
-        if num_delete_keys:
-            region.delete_multi(cache_keys)
-    return num_delete_keys
+def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str):
+    from . import CLEAR_DELETE, CLEAR_INVALIDATE
+
+    if not isinstance(cache_region, RhodeCodeCacheRegion):
+        cache_region = get_or_create_region(cache_region, cache_namespace_uid)
+    log.debug('clearing cache region: %s with method=%s', cache_region, method)
+
+    num_affected_keys = None
+
+    if method == CLEAR_INVALIDATE:
+        # NOTE: The CacheRegion.invalidate() method’s default mode of
+        # operation is to set a timestamp local to this CacheRegion in this Python process only.
+        # It does not impact other Python processes or regions as the timestamp is only stored locally in memory.
+        cache_region.invalidate(hard=True)
+
+    if method == CLEAR_DELETE:
+        cache_keys = cache_region.backend.list_keys(prefix=cache_namespace_uid)
+        num_affected_keys = len(cache_keys)
+        if num_affected_keys:
+            cache_region.delete_multi(cache_keys)
+
+    return num_affected_keys

vcsserver/lib/request_counter.py

@@ -1,7 +1,5 b''
-# -*- coding: utf-8 -*-
-
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/lib/statsd_client.py

@@ -1,3 +1,20 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
 from vcsserver.lib._vendor.statsd import client_from_config
 
 
@@ -12,7 +29,7 b' class _Singleton(type):'
 
     def __call__(cls, *args, **kwargs):
         if cls not in cls._instances:
-            cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 
@@ -24,22 +41,26 b' class StatsdClientClass(Singleton):'
     setup_run = False
     statsd_client = None
     statsd = None
+    strict_mode_init = False
 
     def __getattribute__(self, name):
 
         if name.startswith("statsd"):
             if self.setup_run:
-                return super(StatsdClientClass, self).__getattribute__(name)
+                return super().__getattribute__(name)
             else:
+                if self.strict_mode_init:
+                    raise StatsdClientNotInitialised(f"requested key was {name}")
                 return None
-                #raise StatsdClientNotInitialised("requested key was %s" % name)
 
-        return super(StatsdClientClass, self).__getattribute__(name)
+        return super().__getattribute__(name)
 
     def setup(self, settings):
         """
         Initialize the client
         """
+        strict_init_mode = settings.pop('statsd_strict_init', False)
+
         statsd = client_from_config(settings)
         self.statsd = statsd
         self.statsd_client = statsd

vcsserver/pygrack.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,11 +21,13 b' import os'
 import socket
 import logging
 
-import simplejson as json
 import dulwich.protocol
+from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
 from webob import Request, Response, exc
 
+from vcsserver.lib.rc_json import json
 from vcsserver import hooks, subprocessio
+from vcsserver.str_utils import ascii_bytes
 
 
 log = logging.getLogger(__name__)
@@ -54,7 +56,7 b' class FileWrapper(object):'
         return data
 
     def __repr__(self):
-        return '<FileWrapper %s len: %s, read: %s>' % (
+        return '<FileWrapper {} len: {}, read: {}>'.format(
             self.fd, self.content_length, self.content_length - self.remain
         )
 
@@ -62,26 +64,25 b' class FileWrapper(object):'
 class GitRepository(object):
     """WSGI app for handling Git smart protocol endpoints."""
 
-    git_folder_signature = frozenset(
-        ('config', 'head', 'info', 'objects', 'refs'))
+    git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
     commands = frozenset(('git-upload-pack', 'git-receive-pack'))
-    valid_accepts = frozenset(('application/x-%s-result' %
-                               c for c in commands))
+    valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
 
     # The last bytes are the SHA1 of the first 12 bytes.
     EMPTY_PACK = (
-        'PACK\x00\x00\x00\x02\x00\x00\x00\x00' +
-        '\x02\x9d\x08\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
+        b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
+        b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
     )
-    SIDE_BAND_CAPS = frozenset(('side-band', 'side-band-64k'))
+    FLUSH_PACKET = b"0000"
 
-    def __init__(self, repo_name, content_path, git_path, update_server_info,
-                 extras):
+    SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
+
+    def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
         files = frozenset(f.lower() for f in os.listdir(content_path))
         valid_dir_signature = self.git_folder_signature.issubset(files)
 
         if not valid_dir_signature:
-            raise OSError('%s missing git signature' % content_path)
+            raise OSError(f'{content_path} missing git signature')
 
         self.content_path = content_path
         self.repo_name = repo_name
@@ -123,7 +124,7 b' class GitRepository(object):'
         # It reads binary, per number of bytes specified.
         # if you do add '\n' as part of data, count it.
         server_advert = '# service=%s\n' % git_command
-        packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
+        packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
         try:
             gitenv = dict(os.environ)
             # forget all configs
@@ -133,15 +134,15 b' class GitRepository(object):'
             out = subprocessio.SubprocessIOChunker(
                 command,
                 env=gitenv,
-                starting_values=[packet_len + server_advert + '0000'],
+                starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
                 shell=False
             )
-        except EnvironmentError:
+        except OSError:
             log.exception('Error processing command')
             raise exc.HTTPExpectationFailed()
 
         resp = Response()
-        resp.content_type = 'application/x-%s-advertisement' % str(git_command)
+        resp.content_type = f'application/x-{git_command}-advertisement'
         resp.charset = None
         resp.app_iter = out
 
@@ -166,34 +167,103 b' class GitRepository(object):'
         We also print in the error output a message explaining why the command
         was aborted.
 
-        If aditionally, the user is accepting messages we send them the output
+        If additionally, the user is accepting messages we send them the output
         of the pre-pull hook.
 
         Note that for clients not supporting side-band we just send them the
         emtpy PACK file.
         """
+
         if self.SIDE_BAND_CAPS.intersection(capabilities):
             response = []
             proto = dulwich.protocol.Protocol(None, response.append)
-            proto.write_pkt_line('NAK\n')
-            self._write_sideband_to_proto(pre_pull_messages, proto,
-                                          capabilities)
+            proto.write_pkt_line(dulwich.protocol.NAK_LINE)
+
+            self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
             # N.B.(skreft): Do not change the sideband channel to 3, as that
             # produces a fatal error in the client:
             #   fatal: error in sideband demultiplexer
-            proto.write_sideband(2, 'Pre pull hook failed: aborting\n')
-            proto.write_sideband(1, self.EMPTY_PACK)
+            proto.write_sideband(
+                dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
+                ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
+            proto.write_sideband(
+                dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
+                ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
 
-            # writes 0000
+            # writes b"0000" as default
             proto.write_pkt_line(None)
 
             return response
         else:
-            return [self.EMPTY_PACK]
+            return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
+
+    def _build_post_pull_response(self, response, capabilities, start_message, end_message):
+        """
+        Given a list response we inject the post-pull messages.
+
+        We only inject the messages if the client supports sideband, and the
+        response has the format:
+            0008NAK\n...0000
+
+        Note that we do not check the no-progress capability as by default, git
+        sends it, which effectively would block all messages.
+        """
+
+        if not self.SIDE_BAND_CAPS.intersection(capabilities):
+            return response
+
+        if not start_message and not end_message:
+            return response
+
+        try:
+            iter(response)
+            # iterator probably will work, we continue
+        except TypeError:
+            raise TypeError(f'response must be an iterator: got {type(response)}')
+        if isinstance(response, (list, tuple)):
+            raise TypeError(f'response must be an iterator: got {type(response)}')
+
+        def injected_response():
 
-    def _write_sideband_to_proto(self, data, proto, capabilities):
+            do_loop = 1
+            header_injected = 0
+            next_item = None
+            has_item = False
+            item = b''
+
+            while do_loop:
+
+                try:
+                    next_item = next(response)
+                except StopIteration:
+                    do_loop = 0
+
+                if has_item:
+                    # last item ! alter it now
+                    if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
+                        new_response = [item[:-4]]
+                        new_response.extend(self._get_messages(end_message, capabilities))
+                        new_response.append(self.FLUSH_PACKET)
+                        item = b''.join(new_response)
+
+                    yield item
+
+                has_item = True
+                item = next_item
+
+                # alter item if it's the initial chunk
+                if not header_injected and item.startswith(b'0008NAK\n'):
+                    new_response = [b'0008NAK\n']
+                    new_response.extend(self._get_messages(start_message, capabilities))
+                    new_response.append(item[8:])
+                    item = b''.join(new_response)
+                    header_injected = 1
+
+        return injected_response()
+
+    def _write_sideband_to_proto(self, proto, data, capabilities):
         """
-        Write the data to the proto's sideband number 2.
+        Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
 
         We do not use dulwich's write_sideband directly as it only supports
         side-band-64k.
@@ -204,68 +274,27 b' class GitRepository(object):'
         # N.B.(skreft): The values below are explained in the pack protocol
         # documentation, section Packfile Data.
         # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
-        if 'side-band-64k' in capabilities:
+        if CAPABILITY_SIDE_BAND_64K in capabilities:
             chunk_size = 65515
-        elif 'side-band' in capabilities:
+        elif CAPABILITY_SIDE_BAND in capabilities:
             chunk_size = 995
         else:
             return
 
-        chunker = (
-            data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size))
+        chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
 
         for chunk in chunker:
-            proto.write_sideband(2, chunk)
+            proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
 
     def _get_messages(self, data, capabilities):
         """Return a list with packets for sending data in sideband number 2."""
         response = []
         proto = dulwich.protocol.Protocol(None, response.append)
 
-        self._write_sideband_to_proto(data, proto, capabilities)
+        self._write_sideband_to_proto(proto, data, capabilities)
 
         return response
 
-    def _inject_messages_to_response(self, response, capabilities,
-                                     start_messages, end_messages):
-        """
-        Given a list response we inject the pre/post-pull messages.
-
-        We only inject the messages if the client supports sideband, and the
-        response has the format:
-            0008NAK\n...0000
-
-        Note that we do not check the no-progress capability as by default, git
-        sends it, which effectively would block all messages.
-        """
-        if not self.SIDE_BAND_CAPS.intersection(capabilities):
-            return response
-
-        if not start_messages and not end_messages:
-            return response
-
-        # make a list out of response if it's an iterator
-        # so we can investigate it for message injection.
-        if hasattr(response, '__iter__'):
-            response = list(response)
-
-        if (not response[0].startswith('0008NAK\n') or
-                not response[-1].endswith('0000')):
-            return response
-
-        new_response = ['0008NAK\n']
-        new_response.extend(self._get_messages(start_messages, capabilities))
-        if len(response) == 1:
-            new_response.append(response[0][8:-4])
-        else:
-            new_response.append(response[0][8:])
-            new_response.extend(response[1:-1])
-            new_response.append(response[-1][:-4])
-        new_response.extend(self._get_messages(end_messages, capabilities))
-        new_response.append('0000')
-
-        return new_response
-
     def backend(self, request, environ):
         """
         WSGI Response producer for HTTP POST Git Smart HTTP requests.
@@ -304,14 +333,15 b' class GitRepository(object):'
             inputstream = request.body_file_seekable
 
         resp = Response()
-        resp.content_type = ('application/x-%s-result' %
-                             git_command.encode('utf8'))
+        resp.content_type = f'application/x-{git_command}-result'
         resp.charset = None
 
         pre_pull_messages = ''
+        # Upload-pack == clone
         if git_command == 'git-upload-pack':
-            status, pre_pull_messages = hooks.git_pre_pull(self.extras)
-            if status != 0:
+            hook_response = hooks.git_pre_pull(self.extras)
+            if hook_response.status != 0:
+                pre_pull_messages = hook_response.output
                 resp.app_iter = self._build_failed_pre_pull_response(
                     capabilities, pre_pull_messages)
                 return resp
@@ -326,7 +356,7 b' class GitRepository(object):'
 
         out = subprocessio.SubprocessIOChunker(
             cmd,
-            inputstream=inputstream,
+            input_stream=inputstream,
             env=gitenv,
             cwd=self.content_path,
             shell=False,
@@ -346,7 +376,7 b' class GitRepository(object):'
             log.debug('handling cmd %s', cmd)
             output = subprocessio.SubprocessIOChunker(
                 cmd,
-                inputstream=inputstream,
+                input_stream=inputstream,
                 env=gitenv,
                 cwd=self.content_path,
                 shell=False,
@@ -357,10 +387,11 b' class GitRepository(object):'
             for _ in output:
                 pass
 
+        # Upload-pack == clone
         if git_command == 'git-upload-pack':
-            unused_status, post_pull_messages = hooks.git_post_pull(self.extras)
-            resp.app_iter = self._inject_messages_to_response(
-                out, capabilities, pre_pull_messages, post_pull_messages)
+            hook_response = hooks.git_post_pull(self.extras)
+            post_pull_messages = hook_response.output
+            resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
         else:
             resp.app_iter = out
 

vcsserver/remote/git.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,12 +18,12 b''
 import collections
 import logging
 import os
-import posixpath as vcspath
 import re
 import stat
 import traceback
-import urllib
-import urllib2
+import urllib.request
+import urllib.parse
+import urllib.error
 from functools import wraps
 
 import more_itertools
@@ -31,7 +31,7 b' import pygit2'
 from pygit2 import Repository as LibGit2Repo
 from pygit2 import index as LibGit2Index
 from dulwich import index, objects
-from dulwich.client import HttpGitClient, LocalGitClient
+from dulwich.client import HttpGitClient, LocalGitClient, FetchPackResult
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
@@ -40,8 +40,8 b' from dulwich.repo import Repo as Dulwich'
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
-from vcsserver.utils import safe_str, safe_int, safe_unicode
-from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
+from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes
+from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
@@ -50,19 +50,12 b' from vcsserver.vcs_base import RemoteBas'
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
-PEELED_REF_MARKER = '^{}'
-
+PEELED_REF_MARKER = b'^{}'
+HEAD_MARKER = b'HEAD'
 
 log = logging.getLogger(__name__)
 
 
-def str_to_dulwich(value):
-    """
-    Dulwich 0.10.1a requires `unicode` objects to be passed in.
-    """
-    return value.decode(settings.WIRE_ENCODING)
-
-
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
 
@@ -76,8 +69,8 b' def reraise_safe_exceptions(func):'
         except (HangupException, UnexpectedCommandError) as e:
             exc = exceptions.VcsException(org_exc=e)
             raise exc(safe_str(e))
-        except Exception as e:
-            # NOTE(marcink): becuase of how dulwich handles some exceptions
+        except Exception:
+            # NOTE(marcink): because of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
@@ -114,10 +107,14 b' class GitFactory(RepoFactory):'
 
     def _create_repo(self, wire, create, use_libgit2=False):
         if use_libgit2:
-            return Repository(wire['path'])
+            repo = Repository(safe_bytes(wire['path']))
         else:
-            repo_path = str_to_dulwich(wire['path'])
-            return Repo(repo_path)
+            # dulwich mode
+            repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
+            repo = Repo(repo_path)
+
+        log.debug('repository created: got GIT object: %s', repo)
+        return repo
 
     def repo(self, wire, create=False, use_libgit2=False):
         """
@@ -129,6 +126,28 b' class GitFactory(RepoFactory):'
         return self.repo(wire, use_libgit2=True)
 
 
+def create_signature_from_string(author_str, **kwargs):
+    """
+    Creates a pygit2.Signature object from a string of the format 'Name <email>'.
+
+    :param author_str: String of the format 'Name <email>'
+    :return: pygit2.Signature object
+    """
+    match = re.match(r'^(.+) <(.+)>$', author_str)
+    if match is None:
+        raise ValueError(f"Invalid format: {author_str}")
+
+    name, email = match.groups()
+    return pygit2.Signature(name, email, **kwargs)
+
+
+def get_obfuscated_url(url_obj):
+    url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
+    url_obj.query = obfuscate_qs(url_obj.query)
+    obfuscated_uri = str(url_obj)
+    return obfuscated_uri
+
+
 class GitRemote(RemoteBase):
 
     def __init__(self, factory):
@@ -141,10 +160,17 b' class GitRemote(RemoteBase):'
             "parents": self.parents,
             "_commit": self.revision,
         }
+        self._bulk_file_methods = {
+            "size": self.get_node_size,
+            "data": self.get_node_data,
+            "flags": self.get_node_flags,
+            "is_binary": self.get_node_is_binary,
+            "md5": self.md5_hash
+        }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
-            return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
+            return {x[0] + '_' + x[1]: x[2] for x in wire['config']}
         return {}
 
     def _remote_conf(self, config):
@@ -153,17 +179,17 b' class GitRemote(RemoteBase):'
         ]
         ssl_cert_dir = config.get('vcs_ssl_dir')
         if ssl_cert_dir:
-            params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
+            params.extend(['-c', f'http.sslCAinfo={ssl_cert_dir}'])
         return params
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
-        prefix = 'git version'
+        prefix = b'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
-        return stdout.strip()
+        return safe_str(stdout.strip())
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
@@ -186,20 +212,27 b' class GitRemote(RemoteBase):'
     def assert_correct_path(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _assert_correct_path(_context_uid, _repo_id):
-            try:
-                repo_init = self._factory.repo_libgit2(wire)
-                with repo_init as repo:
-                    pass
-            except pygit2.GitError:
-                path = wire.get('path')
-                tb = traceback.format_exc()
-                log.debug("Invalid Git path `%s`, tb: %s", path, tb)
+        def _assert_correct_path(_context_uid, _repo_id, fast_check):
+            if fast_check:
+                path = safe_str(wire['path'])
+                if pygit2.discover_repository(path):
+                    return True
                 return False
+            else:
+                try:
+                    repo_init = self._factory.repo_libgit2(wire)
+                    with repo_init:
+                        pass
+                except pygit2.GitError:
+                    path = wire.get('path')
+                    tb = traceback.format_exc()
+                    log.debug("Invalid Git path `%s`, tb: %s", path, tb)
+                    return False
+                return True
 
-            return True
-        return _assert_correct_path(context_uid, repo_id)
+        return _assert_correct_path(context_uid, repo_id, True)
 
     @reraise_safe_exceptions
     def bare(self, wire):
@@ -208,17 +241,69 b' class GitRemote(RemoteBase):'
             return repo.is_bare
 
     @reraise_safe_exceptions
+    def get_node_data(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return BytesEnvelope(blob_obj.data)
+
+    @reraise_safe_exceptions
+    def get_node_size(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.size
+
+    @reraise_safe_exceptions
+    def get_node_flags(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.filemode
+
+    @reraise_safe_exceptions
+    def get_node_is_binary(self, wire, commit_id, path):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blob_obj = commit.tree[path]
+
+            if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                raise exceptions.LookupException()(
+                    f'Tree for commit_id:{commit_id} is not a blob: {blob_obj.type_str}')
+
+            return blob_obj.is_binary
+
+    @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob_obj = repo[sha]
-            blob = blob_obj.data
-            return blob
+            return BytesEnvelope(blob_obj.data)
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _blob_raw_length(_repo_id, _sha):
 
@@ -230,10 +315,10 b' class GitRemote(RemoteBase):'
         return _blob_raw_length(repo_id, sha)
 
     def _parse_lfs_pointer(self, raw_content):
+        spec_string = b'version https://git-lfs.github.com/spec'
+        if raw_content and raw_content.startswith(spec_string):
 
-        spec_string = 'version https://git-lfs.github.com/spec'
-        if raw_content and raw_content.startswith(spec_string):
-            pattern = re.compile(r"""
+            pattern = re.compile(rb"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
@@ -249,8 +334,8 b' class GitRemote(RemoteBase):'
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
@@ -266,8 +351,8 b' class GitRemote(RemoteBase):'
     @reraise_safe_exceptions
     def is_binary(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _tree_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -278,6 +363,26 b' class GitRemote(RemoteBase):'
         return _is_binary(repo_id, tree_id)
 
     @reraise_safe_exceptions
+    def md5_hash(self, wire, commit_id, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _md5_hash(_repo_id, _commit_id, _path):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[_commit_id]
+                blob_obj = commit.tree[_path]
+
+                if blob_obj.type != pygit2.GIT_OBJ_BLOB:
+                    raise exceptions.LookupException()(
+                        f'Tree for commit_id:{_commit_id} is not a blob: {blob_obj.type_str}')
+
+                return ''
+
+        return _md5_hash(repo_id, commit_id, path)
+
+    @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
@@ -305,90 +410,105 b' class GitRemote(RemoteBase):'
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
-        raise ValueError('Unable to fetch oid with path {}'.format(oid))
+        raise ValueError(f'Unable to fetch oid with path {oid}')
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _rev, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
                     args = [wire, rev]
                     result[attr] = method(*args)
                 except KeyError as e:
-                    raise exceptions.VcsException(e)(
-                        "Unknown bulk attribute: %s" % attr)
+                    raise exceptions.VcsException(e)(f"Unknown bulk attribute: {attr}")
             return result
 
         return _bulk_request(repo_id, rev, sorted(pre_load))
 
-    def _build_opener(self, url):
+    @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
+
+    def _build_opener(self, url: str):
         handlers = []
-        url_obj = url_parser(url)
-        _, authinfo = url_obj.authinfo()
+        url_obj = url_parser(safe_bytes(url))
+        authinfo = url_obj.authinfo()[1]
 
         if authinfo:
             # create a password manager
-            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
-        return urllib2.build_opener(*handlers)
-
-    def _type_id_to_name(self, type_id):
-        return {
-            1: b'commit',
-            2: b'tree',
-            3: b'blob',
-            4: b'tag'
-        }[type_id]
+        return urllib.request.build_opener(*handlers)
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
-        url_obj = url_parser(url)
-        test_uri, _ = url_obj.authinfo()
-        url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
-        url_obj.query = obfuscate_qs(url_obj.query)
-        cleaned_uri = str(url_obj)
-        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
+        url_obj = url_parser(safe_bytes(url))
+
+        test_uri = safe_str(url_obj.authinfo()[0])
+        obfuscated_uri = get_obfuscated_url(url_obj)
+
+        log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
-        o = self._build_opener(url)
+        o = self._build_opener(test_uri)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
-        qs = '?%s' % urllib.urlencode(q)
-        cu = "%s%s" % (test_uri, qs)
-        req = urllib2.Request(cu, None, {})
+        qs = '?%s' % urllib.parse.urlencode(q)
+        cu = f"{test_uri}{qs}"
+        req = urllib.request.Request(cu, None, {})
 
         try:
-            log.debug("Trying to open URL %s", cleaned_uri)
+            log.debug("Trying to open URL %s", obfuscated_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
-            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
+            log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
             # means it cannot be cloned
-            raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
+            raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
 
         # now detect if it's proper git repo
-        gitdata = resp.read()
-        if 'service=git-upload-pack' in gitdata:
+        gitdata: bytes = resp.read()
+
+        if b'service=git-upload-pack' in gitdata:
             pass
-        elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
+        elif re.findall(br'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
-            raise exceptions.URLError()(
-                "url [%s] does not look like an git" % (cleaned_uri,))
+            e = None
+            raise exceptions.URLError(e)(
+                "url [%s] does not look like an hg repo org_exc: %s"
+                % (obfuscated_uri, e))
 
         return True
 
@@ -415,6 +535,7 b' class GitRemote(RemoteBase):'
     def branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _branch(_context_uid, _repo_id, _commit_id):
             regex = re.compile('^refs/heads')
@@ -422,7 +543,7 b' class GitRemote(RemoteBase):'
             def filter_with(ref):
                 return regex.match(ref[0]) and ref[1] == _commit_id
 
-            branches = filter(filter_with, self.get_refs(wire).items())
+            branches = list(filter(filter_with, list(self.get_refs(wire).items())))
             return [x[0].split('refs/heads/')[-1] for x in branches]
 
         return _branch(context_uid, repo_id, commit_id)
@@ -431,6 +552,7 b' class GitRemote(RemoteBase):'
     def commit_branches(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _commit_branches(_context_uid, _repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -449,152 +571,136 b' class GitRemote(RemoteBase):'
             repo.object_store.add_object(blob)
             return blob.id
 
-    # TODO: this is quite complex, check if that can be simplified
+    @reraise_safe_exceptions
+    def create_commit(self, wire, author, committer, message, branch, new_tree_id, date_args: list[int, int] = None):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+
+            if date_args:
+                current_time, offset = date_args
+
+                kw = {
+                    'time': current_time,
+                    'offset': offset
+                }
+            author = create_signature_from_string(author, **kw)
+            committer = create_signature_from_string(committer, **kw)
+
+            tree = new_tree_id
+            if isinstance(tree, (bytes, str)):
+                # validate this tree is in the repo...
+                tree = repo[safe_str(tree)].id
+
+            parents = []
+            # ensure we COMMIT on top of given branch head
+            # check if this repo has ANY branches, otherwise it's a new branch case we need to make
+            if branch in repo.branches.local:
+                parents += [repo.branches[branch].target]
+            elif [x for x in repo.branches.local]:
+                parents += [repo.head.target]
+            #else:
+                # in case we want to commit on new branch we create it on top of HEAD
+                #repo.branches.local.create(branch, repo.revparse_single('HEAD'))
+
+            # # Create a new commit
+            commit_oid = repo.create_commit(
+                f'refs/heads/{branch}',  # the name of the reference to update
+                author,  # the author of the commit
+                committer,  # the committer of the commit
+                message,  # the commit message
+                tree,  # the tree produced by the index
+                parents  # list of parents for the new commit, usually just one,
+            )
+
+            new_commit_id = safe_str(commit_oid)
+
+            return new_commit_id
+
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
-        # Defines the root tree
-        class _Root(object):
-            def __repr__(self):
-                return 'ROOT TREE'
-        ROOT = _Root()
 
-        repo = self._factory.repo(wire)
-        object_store = repo.object_store
-
-        # Create tree and populates it with blobs
-
-        if commit_tree and repo[commit_tree]:
-            git_commit = repo[commit_data['parents'][0]]
-            commit_tree = repo[git_commit.tree]  # root tree
-        else:
-            commit_tree = objects.Tree()
-
-        for node in updated:
-            # Compute subdirs if needed
-            dirpath, nodename = vcspath.split(node['path'])
-            dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
-            parent = commit_tree
-            ancestors = [('', parent)]
+        def mode2pygit(mode):
+            """
+            git only supports two filemode 644 and 755
 
-            # Tries to dig for the deepest existing tree
-            while dirnames:
-                curdir = dirnames.pop(0)
-                try:
-                    dir_id = parent[curdir][1]
-                except KeyError:
-                    # put curdir back into dirnames and stops
-                    dirnames.insert(0, curdir)
-                    break
-                else:
-                    # If found, updates parent
-                    parent = repo[dir_id]
-                    ancestors.append((curdir, parent))
-            # Now parent is deepest existing tree and we need to create
-            # subtrees for dirnames (in reverse order)
-            # [this only applies for nodes from added]
-            new_trees = []
+            0o100755 -> 33261
+            0o100644 -> 33188
+            """
+            return {
+                0o100644: pygit2.GIT_FILEMODE_BLOB,
+                0o100755: pygit2.GIT_FILEMODE_BLOB_EXECUTABLE,
+                0o120000: pygit2.GIT_FILEMODE_LINK
+            }.get(mode) or pygit2.GIT_FILEMODE_BLOB
 
-            blob = objects.Blob.from_string(node['content'])
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo_index = repo.index
 
-            if dirnames:
-                # If there are trees which should be created we need to build
-                # them now (in reverse order)
-                reversed_dirnames = list(reversed(dirnames))
-                curtree = objects.Tree()
-                curtree[node['node_path']] = node['mode'], blob.id
-                new_trees.append(curtree)
-                for dirname in reversed_dirnames[:-1]:
-                    newtree = objects.Tree()
-                    newtree[dirname] = (DIR_STAT, curtree.id)
-                    new_trees.append(newtree)
-                    curtree = newtree
-                parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
-            else:
-                parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
+            for pathspec in updated:
+                blob_id = repo.create_blob(pathspec['content'])
+                ie = pygit2.IndexEntry(pathspec['path'], blob_id, mode2pygit(pathspec['mode']))
+                repo_index.add(ie)
 
-            new_trees.append(parent)
-            # Update ancestors
-            reversed_ancestors = reversed(
-                [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
-            for parent, tree, path in reversed_ancestors:
-                parent[path] = (DIR_STAT, tree.id)
-                object_store.add_object(tree)
+            for pathspec in removed:
+                repo_index.remove(pathspec)
 
-            object_store.add_object(blob)
-            for tree in new_trees:
-                object_store.add_object(tree)
+            # Write changes to the index
+            repo_index.write()
+
+            # Create a tree from the updated index
+            commit_tree = repo_index.write_tree()
+
+        new_tree_id = commit_tree
 
-        for node_path in removed:
-            paths = node_path.split('/')
-            tree = commit_tree  # start with top-level
-            trees = [{'tree': tree, 'path': ROOT}]
-            # Traverse deep into the forest...
-            # resolve final tree by iterating the path.
-            # e.g a/b/c.txt will get
-            # - root as tree then
-            # - 'a' as tree,
-            # - 'b' as tree,
-            # - stop at c as blob.
-            for path in paths:
-                try:
-                    obj = repo[tree[path][1]]
-                    if isinstance(obj, objects.Tree):
-                        trees.append({'tree': obj, 'path': path})
-                        tree = obj
-                except KeyError:
-                    break
-            #PROBLEM:
-            """
-            We're not editing same reference tree object
-            """
-            # Cut down the blob and all rotten trees on the way back...
-            for path, tree_data in reversed(zip(paths, trees)):
-                tree = tree_data['tree']
-                tree.__delitem__(path)
-                # This operation edits the tree, we need to mark new commit back
+        author = commit_data['author']
+        committer = commit_data['committer']
+        message = commit_data['message']
+
+        date_args = [int(commit_data['commit_time']), int(commit_data['commit_timezone'])]
 
-                if len(tree) > 0:
-                    # This tree still has elements - don't remove it or any
-                    # of it's parents
-                    break
-
-        object_store.add_object(commit_tree)
+        new_commit_id = self.create_commit(wire, author, committer, message, branch,
+                                           new_tree_id, date_args=date_args)
 
-        # Create commit
-        commit = objects.Commit()
-        commit.tree = commit_tree.id
-        for k, v in commit_data.items():
-            setattr(commit, k, v)
-        object_store.add_object(commit)
+        # libgit2, ensure the branch is there and exists
+        self.create_branch(wire, branch, new_commit_id)
 
-        self.create_branch(wire, branch, commit.id)
+        # libgit2, set new ref to this created commit
+        self.set_refs(wire, f'refs/heads/{branch}', new_commit_id)
 
-        # dulwich set-ref
-        ref = 'refs/heads/%s' % branch
-        repo.refs[ref] = commit.id
-
-        return commit.id
+        return new_commit_id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
-            url_obj = url_parser(url)
+            url_obj = url_parser(safe_bytes(url))
             o = self._build_opener(url)
-            url, _ = url_obj.authinfo()
+            url = url_obj.authinfo()[0]
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
-            def determine_wants_requested(references):
-                return [references[r] for r in references if r in refs]
+            refs = [ascii_bytes(x) for x in refs]
+
+            def determine_wants_requested(remote_refs):
+                determined = []
+                for ref_name, ref_hash in remote_refs.items():
+                    bytes_ref_name = safe_bytes(ref_name)
+
+                    if bytes_ref_name in refs:
+                        bytes_ref_hash = safe_bytes(ref_hash)
+                        determined.append(bytes_ref_hash)
+                return determined
+
+            # swap with our custom requested wants
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
+
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
@@ -619,18 +725,21 b' class GitRemote(RemoteBase):'
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
-                repo["HEAD"] = remote_refs[refs[-1]]
+                repo[HEAD_MARKER] = remote_refs[refs[-1]]
 
         if update_after:
-            # we want to checkout HEAD
-            repo["HEAD"] = remote_refs["HEAD"]
+            # we want to check out HEAD
+            repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
             index.build_index_from_tree(repo.path, repo.index_path(),
-                                        repo.object_store, repo["HEAD"].tree)
+                                        repo.object_store, repo[HEAD_MARKER].tree)
+
+        if isinstance(remote_refs, FetchPackResult):
+            return remote_refs.refs
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_fetch(self, wire, url, refs=None, all_refs=False):
-        repo = self._factory.repo(wire)
+        self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
 
@@ -649,7 +758,7 b' class GitRemote(RemoteBase):'
         fetch_refs = []
 
         for ref_line in output.splitlines():
-            sha, ref = ref_line.split('\t')
+            sha, ref = ref_line.split(b'\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
@@ -658,23 +767,23 b' class GitRemote(RemoteBase):'
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
-            if ref in ['HEAD']:
+            if ref in [HEAD_MARKER]:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
-                fetch_refs.append('{}:{}'.format(ref, ref))
+                fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
             elif not refs:
-                fetch_refs.append('{}:{}'.format(ref, ref))
+                fetch_refs.append(f'{safe_str(ref)}:{safe_str(ref)}')
         log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
 
         if fetch_refs:
             for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
                 fetch_refs_chunks = list(chunk)
                 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
-                _out, _err = self.run_git_command(
+                self.run_git_command(
                     wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
@@ -711,17 +820,18 b' class GitRemote(RemoteBase):'
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
-        LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
+        LocalGitClient(thin_packs=False).fetch(path2, repo_remote)
 
         revs = [
             x.commit.id
-            for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
+            for x in repo_remote.get_walker(include=[safe_bytes(rev2)], exclude=[safe_bytes(rev1)])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha, maybe_unreachable=False):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_object(_context_uid, _repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
@@ -766,11 +876,11 b' class GitRemote(RemoteBase):'
                         raise exceptions.LookupException(e)(missing_commit_err)
 
                 commit_id = commit.hex
-                type_id = commit.type
+                type_str = commit.type_str
 
                 return {
                     'id': commit_id,
-                    'type': self._type_id_to_name(type_id),
+                    'type': type_str,
                     'commit_id': commit_id,
                     'idx': 0
                 }
@@ -781,6 +891,7 b' class GitRemote(RemoteBase):'
     def get_refs(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_refs(_context_uid, _repo_id):
 
@@ -788,7 +899,7 b' class GitRemote(RemoteBase):'
             with repo_init as repo:
                 regex = re.compile('^refs/(heads|tags)/')
                 return {x.name: x.target.hex for x in
-                        filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
+                        [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]}
 
         return _get_refs(context_uid, repo_id)
 
@@ -796,13 +907,14 b' class GitRemote(RemoteBase):'
     def get_branch_pointers(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_branch_pointers(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             regex = re.compile('^refs/heads')
             with repo_init as repo:
-                branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
+                branches = [ref for ref in repo.listall_reference_objects() if regex.match(ref.name)]
                 return {x.target.hex: x.shorthand for x in branches}
 
         return _get_branch_pointers(context_uid, repo_id)
@@ -811,6 +923,7 b' class GitRemote(RemoteBase):'
     def head(self, wire, show_exc=True):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _head(_context_uid, _repo_id, _show_exc):
             repo_init = self._factory.repo_libgit2(wire)
@@ -824,12 +937,12 b' class GitRemote(RemoteBase):'
 
     @reraise_safe_exceptions
     def init(self, wire):
-        repo_path = str_to_dulwich(wire['path'])
+        repo_path = safe_str(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
-        repo_path = str_to_dulwich(wire['path'])
+        repo_path = safe_str(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
@@ -837,6 +950,7 b' class GitRemote(RemoteBase):'
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _revision(_context_uid, _repo_id, _rev):
             repo_init = self._factory.repo_libgit2(wire)
@@ -856,6 +970,7 b' class GitRemote(RemoteBase):'
     def date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _date(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -876,6 +991,7 b' class GitRemote(RemoteBase):'
     def author(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _author(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -888,12 +1004,12 b' class GitRemote(RemoteBase):'
                     author = commit.get_object().author
 
                 if author.email:
-                    return u"{} <{}>".format(author.name, author.email)
+                    return f"{author.name} <{author.email}>"
 
                 try:
-                    return u"{}".format(author.name)
+                    return f"{author.name}"
                 except Exception:
-                    return u"{}".format(safe_unicode(author.raw_name))
+                    return f"{safe_str(author.raw_name)}"
 
         return _author(repo_id, commit_id)
 
@@ -901,6 +1017,7 b' class GitRemote(RemoteBase):'
     def message(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _message(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -913,6 +1030,7 b' class GitRemote(RemoteBase):'
     def parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _parents(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
@@ -930,17 +1048,23 b' class GitRemote(RemoteBase):'
     def children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
+        head = self.head(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _children(_repo_id, _commit_id):
+
             output, __ = self.run_git_command(
-                wire, ['rev-list', '--all', '--children'])
+                wire, ['rev-list', '--all', '--children', f'{commit_id}^..{head}'])
 
             child_ids = []
-            pat = re.compile(r'^%s' % commit_id)
-            for l in output.splitlines():
-                if pat.match(l):
-                    found_ids = l.split(' ')[1:]
+            pat = re.compile(fr'^{commit_id}')
+            for line in output.splitlines():
+                line = safe_str(line)
+                if pat.match(line):
+                    found_ids = line.split(' ')[1:]
                     child_ids.extend(found_ids)
+                    break
 
             return child_ids
         return _children(repo_id, commit_id)
@@ -955,7 +1079,11 b' class GitRemote(RemoteBase):'
     def create_branch(self, wire, branch_name, commit_id, force=False):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
-            commit = repo[commit_id]
+            if commit_id:
+                commit = repo[commit_id]
+            else:
+                # if commit is not given  just use the HEAD
+                commit = repo.head()
 
             if force:
                 repo.branches.local.create(branch_name, commit, force=force)
@@ -973,23 +1101,39 b' class GitRemote(RemoteBase):'
     def tag_remove(self, wire, tag_name):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
-            key = 'refs/tags/{}'.format(tag_name)
+            key = f'refs/tags/{tag_name}'
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
-        # TODO(marcink): remove this seems it's only used by tests
         repo = self._factory.repo(wire)
+        # source can be empty
+        source_id = safe_bytes(source_id if source_id else b'')
+        target_id = safe_bytes(target_id)
+
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
-        return list(result)
+
+        added = set()
+        modified = set()
+        deleted = set()
+        for (old_path, new_path), (_, _), (_, _) in list(result):
+            if new_path and old_path:
+                modified.add(new_path)
+            elif new_path and not old_path:
+                added.add(new_path)
+            elif not new_path and old_path:
+                deleted.add(old_path)
+
+        return list(added), list(modified), list(deleted)
 
     @reraise_safe_exceptions
     def tree_and_type_for_path(self, wire, commit_id, path):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
             repo_init = self._factory.repo_libgit2(wire)
@@ -1001,13 +1145,14 b' class GitRemote(RemoteBase):'
                 except KeyError:
                     return None, None, None
 
-                return tree.id.hex, tree.type, tree.filemode
+                return tree.id.hex, tree.type_str, tree.filemode
         return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_items(_repo_id, _tree_id):
 
@@ -1016,13 +1161,13 b' class GitRemote(RemoteBase):'
                 try:
                     tree = repo[tree_id]
                 except KeyError:
-                    raise ObjectMissing('No tree with id: {}'.format(tree_id))
+                    raise ObjectMissing(f'No tree with id: {tree_id}')
 
                 result = []
                 for item in tree:
                     item_sha = item.hex
                     item_mode = item.filemode
-                    item_type = item.type
+                    item_type = item.type_str
 
                     if item_type == 'commit':
                         # NOTE(marcink): submodules we translate to 'link' for backward compat
@@ -1066,7 +1211,7 b' class GitRemote(RemoteBase):'
             lines = diff.splitlines()
             x = 0
             for line in lines:
-                if line.startswith('diff'):
+                if line.startswith(b'diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
@@ -1076,6 +1221,7 b' class GitRemote(RemoteBase):'
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         repo_init = self._factory.repo_libgit2(wire)
+
         with repo_init as repo:
             swap = True
             flags = 0
@@ -1101,15 +1247,17 b' class GitRemote(RemoteBase):'
             if file_filter:
                 for p in diff_obj:
                     if p.delta.old_file.path == file_filter:
-                        return p.patch or ''
+                        return BytesEnvelope(p.data) or BytesEnvelope(b'')
                 # fo matching path == no diff
-                return ''
-            return diff_obj.patch or ''
+                return BytesEnvelope(b'')
+
+            return BytesEnvelope(safe_bytes(diff_obj.patch)) or BytesEnvelope(b'')
 
     @reraise_safe_exceptions
     def node_history(self, wire, commit_id, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
             # optimize for n==1, rev-list is much faster for that use-case
@@ -1122,14 +1270,14 b' class GitRemote(RemoteBase):'
                 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
 
             output, __ = self.run_git_command(wire, cmd)
-            commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
+            commit_ids = re.findall(rb'[0-9a-fA-F]{40}', output)
 
             return [x for x in commit_ids]
         return _node_history(context_uid, repo_id, commit_id, path, limit)
 
     @reraise_safe_exceptions
-    def node_annotate(self, wire, commit_id, path):
-
+    def node_annotate_legacy(self, wire, commit_id, path):
+        # note: replaced by pygit2 implementation
         cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for boundaries
@@ -1137,13 +1285,31 b' class GitRemote(RemoteBase):'
         output, __ = self.run_git_command(wire, cmd)
 
         result = []
-        for i, blame_line in enumerate(output.split('\n')[:-1]):
+        for i, blame_line in enumerate(output.splitlines()[:-1]):
             line_no = i + 1
-            commit_id, line = re.split(r' ', blame_line, 1)
-            result.append((line_no, commit_id, line))
+            blame_commit_id, line = re.split(rb' ', blame_line, 1)
+            result.append((line_no, blame_commit_id, line))
+
         return result
 
     @reraise_safe_exceptions
+    def node_annotate(self, wire, commit_id, path):
+
+        result_libgit = []
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+            blame_obj = repo.blame(path, newest_commit=commit_id)
+            for i, line in enumerate(commit.tree[path].data.splitlines()):
+                line_no = i + 1
+                hunk = blame_obj.for_line(line_no)
+                blame_commit_id = hunk.final_commit_id.hex
+
+                result_libgit.append((line_no, blame_commit_id, line))
+
+        return result_libgit
+
+    @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
@@ -1153,6 +1319,7 b' class GitRemote(RemoteBase):'
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id):
 
@@ -1163,6 +1330,16 b' class GitRemote(RemoteBase):'
             except Exception:
                 # Can be raised for empty repositories
                 return []
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_all_commit_ids_pygit2(_context_uid, _repo_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            from pygit2 import GIT_SORT_REVERSE, GIT_SORT_TIME, GIT_BRANCH_ALL
+            results = []
+            with repo_init as repo:
+                for commit in repo.walk(repo.head.target, GIT_SORT_TIME | GIT_BRANCH_ALL | GIT_SORT_REVERSE):
+                    results.append(commit.id.hex)
+
         return _get_all_commit_ids(context_uid, repo_id)
 
     @reraise_safe_exceptions
@@ -1176,7 +1353,7 b' class GitRemote(RemoteBase):'
             _copts = []
             del opts['_bare']
         else:
-            _copts = ['-c', 'core.quotepath=false', ]
+            _copts = ['-c', 'core.quotepath=false',]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
@@ -1203,9 +1380,9 b' class GitRemote(RemoteBase):'
             _opts.update(opts)
             proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
-            return ''.join(proc), ''.join(proc.error)
-        except (EnvironmentError, OSError) as err:
-            cmd = ' '.join(cmd)  # human friendly CMD
+            return b''.join(proc), b''.join(proc.stderr)
+        except OSError as err:
+            cmd = ' '.join(map(safe_str, cmd))  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
@@ -1224,6 +1401,9 b' class GitRemote(RemoteBase):'
         from vcsserver.hook_utils import install_git_hooks
         bare = self.bare(wire)
         path = wire['path']
+        binary_dir = settings.BINARY_DIR
+        if binary_dir:
+            os.path.join(binary_dir, 'python3')
         return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions
@@ -1240,13 +1420,15 b' class GitRemote(RemoteBase):'
     @reraise_safe_exceptions
     def set_head_ref(self, wire, head_name):
         log.debug('Setting refs/head to `%s`', head_name)
-        cmd = ['symbolic-ref', '"HEAD"', '"refs/heads/%s"' % head_name]
-        output, __ = self.run_git_command(wire, cmd)
-        return [head_name] + output.splitlines()
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo.set_head(f'refs/heads/{head_name}')
+
+        return [head_name] + [f'set HEAD to refs/heads/{head_name}']
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def file_walker(_commit_id, path):
             repo_init = self._factory.repo_libgit2(wire)
@@ -1262,20 +1444,20 b' class GitRemote(RemoteBase):'
                     try:
                         tree = repo[tree_id]
                     except KeyError:
-                        raise ObjectMissing('No tree with id: {}'.format(tree_id))
+                        raise ObjectMissing(f'No tree with id: {tree_id}')
 
                 index = LibGit2Index.Index()
                 index.read_tree(tree)
                 file_iter = index
 
-                for fn in file_iter:
-                    file_path = fn.path
-                    mode = fn.mode
+                for file_node in file_iter:
+                    file_path = file_node.path
+                    mode = file_node.mode
                     is_link = stat.S_ISLNK(mode)
                     if mode == pygit2.GIT_FILEMODE_COMMIT:
                         log.debug('Skipping path %s as a commit node', file_path)
                         continue
-                    yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
+                    yield ArchiveNode(file_path, mode, is_link, repo[file_node.hex].read_raw)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)

vcsserver/remote/hg.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,17 +14,17 b''
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
-import functools
+import binascii
 import io
 import logging
-import os
 import stat
-import urllib
-import urllib2
+import urllib.request
+import urllib.parse
 import traceback
+import hashlib
 
 from hgext import largefiles, rebase, purge
-from hgext.strip import strip as hgext_strip
+
 from mercurial import commands
 from mercurial import unionrepo
 from mercurial import verify
@@ -32,15 +32,19 b' from mercurial import repair'
 
 import vcsserver
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, archive_repo, ArchiveNode
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original, store_archive_in_cache, ArchiveNode, BytesEnvelope, \
+    BinaryEnvelope
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError,
-    alwaysmatcher, patternmatcher, hgutil)
+    alwaysmatcher, patternmatcher, hgutil, hgext_strip)
+from vcsserver.str_utils import ascii_bytes, ascii_str, safe_str, safe_bytes
 from vcsserver.vcs_base import RemoteBase
+from vcsserver.config import hooks as hooks_config
+
 
 log = logging.getLogger(__name__)
 
@@ -48,25 +52,31 b' log = logging.getLogger(__name__)'
 def make_ui_from_config(repo_config):
 
     class LoggingUI(ui.ui):
+
         def status(self, *msg, **opts):
-            log.info(' '.join(msg).rstrip('\n'))
-            super(LoggingUI, self).status(*msg, **opts)
+            str_msg = map(safe_str, msg)
+            log.info(' '.join(str_msg).rstrip('\n'))
+            #super(LoggingUI, self).status(*msg, **opts)
 
         def warn(self, *msg, **opts):
-            log.warn(' '.join(msg).rstrip('\n'))
-            super(LoggingUI, self).warn(*msg, **opts)
+            str_msg = map(safe_str, msg)
+            log.warning('ui_logger:'+' '.join(str_msg).rstrip('\n'))
+            #super(LoggingUI, self).warn(*msg, **opts)
 
         def error(self, *msg, **opts):
-            log.error(' '.join(msg).rstrip('\n'))
-            super(LoggingUI, self).error(*msg, **opts)
+            str_msg = map(safe_str, msg)
+            log.error('ui_logger:'+' '.join(str_msg).rstrip('\n'))
+            #super(LoggingUI, self).error(*msg, **opts)
 
         def note(self, *msg, **opts):
-            log.info(' '.join(msg).rstrip('\n'))
-            super(LoggingUI, self).note(*msg, **opts)
+            str_msg = map(safe_str, msg)
+            log.info('ui_logger:'+' '.join(str_msg).rstrip('\n'))
+            #super(LoggingUI, self).note(*msg, **opts)
 
         def debug(self, *msg, **opts):
-            log.debug(' '.join(msg).rstrip('\n'))
-            super(LoggingUI, self).debug(*msg, **opts)
+            str_msg = map(safe_str, msg)
+            log.debug('ui_logger:'+' '.join(str_msg).rstrip('\n'))
+            #super(LoggingUI, self).debug(*msg, **opts)
 
     baseui = LoggingUI()
 
@@ -76,26 +86,26 b' def make_ui_from_config(repo_config):'
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
-        baseui.setconfig(section, option, value)
+        baseui.setconfig(ascii_bytes(section), ascii_bytes(option), ascii_bytes(value))
 
     # make our hgweb quiet so it doesn't print output
-    baseui.setconfig('ui', 'quiet', 'true')
+    baseui.setconfig(b'ui', b'quiet', b'true')
 
-    baseui.setconfig('ui', 'paginate', 'never')
+    baseui.setconfig(b'ui', b'paginate', b'never')
     # for better Error reporting of Mercurial
-    baseui.setconfig('ui', 'message-output', 'stderr')
+    baseui.setconfig(b'ui', b'message-output', b'stderr')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
-    baseui.setconfig('worker', 'numcpus', 1)
+    baseui.setconfig(b'worker', b'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
-    if not baseui.hasconfig('extensions', 'largefiles'):
+    if not baseui.hasconfig(b'extensions', b'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
-        baseui.setconfig('extensions', 'largefiles', '!')
+        baseui.setconfig(b'extensions', b'largefiles', b'!')
 
     return baseui
 
@@ -107,19 +117,19 b' def reraise_safe_exceptions(func):'
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired) as e:
-            raise_from_original(exceptions.AbortException(e))
+            raise_from_original(exceptions.AbortException(e), e)
         except RepoLookupError as e:
-            raise_from_original(exceptions.LookupException(e))
+            raise_from_original(exceptions.LookupException(e), e)
         except RequirementError as e:
-            raise_from_original(exceptions.RequirementException(e))
+            raise_from_original(exceptions.RequirementException(e), e)
         except RepoError as e:
-            raise_from_original(exceptions.VcsException(e))
+            raise_from_original(exceptions.VcsException(e), e)
         except LookupError as e:
-            raise_from_original(exceptions.LookupException(e))
+            raise_from_original(exceptions.LookupException(e), e)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
-                raise_from_original(exceptions.UnhandledException(e))
+                raise_from_original(exceptions.UnhandledException(e), e)
 
             raise
     return wrapper
@@ -130,9 +140,18 b' class MercurialFactory(RepoFactory):'
 
     def _create_config(self, config, hooks=True):
         if not hooks:
-            hooks_to_clean = frozenset((
-                'changegroup.repo_size', 'preoutgoing.pre_pull',
-                'outgoing.pull_logger', 'prechangegroup.pre_push'))
+
+            hooks_to_clean = {
+
+                hooks_config.HOOK_REPO_SIZE,
+                hooks_config.HOOK_PRE_PULL,
+                hooks_config.HOOK_PULL,
+
+                hooks_config.HOOK_PRE_PUSH,
+                # TODO: what about PRETXT, this was disabled in pre 5.0.0
+                hooks_config.HOOK_PRETX_PUSH,
+
+            }
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
@@ -145,7 +164,9 b' class MercurialFactory(RepoFactory):'
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
-        return instance(baseui, wire["path"], create)
+        repo = instance(baseui, safe_bytes(wire["path"]), create)
+        log.debug('repository created: got HG object: %s', repo)
+        return repo
 
     def repo(self, wire, create=False):
         """
@@ -155,7 +176,7 b' class MercurialFactory(RepoFactory):'
 
 
 def patch_ui_message_output(baseui):
-    baseui.setconfig('ui', 'quiet', 'false')
+    baseui.setconfig(b'ui', b'quiet', b'false')
     output = io.BytesIO()
 
     def write(data, **unused_kwargs):
@@ -169,6 +190,22 b' def patch_ui_message_output(baseui):'
     return baseui, output
 
 
+def get_obfuscated_url(url_obj):
+    url_obj.passwd = b'*****' if url_obj.passwd else url_obj.passwd
+    url_obj.query = obfuscate_qs(url_obj.query)
+    obfuscated_uri = str(url_obj)
+    return obfuscated_uri
+
+
+def normalize_url_for_hg(url: str):
+    _proto = None
+
+    if '+' in url[:url.find('://')]:
+        _proto = url[0:url.find('+')]
+        url = url[url.find('+') + 1:]
+    return url, _proto
+
+
 class HgRemote(RemoteBase):
 
     def __init__(self, factory):
@@ -187,6 +224,13 b' class HgRemote(RemoteBase):'
             "hidden": self.ctx_hidden,
             "_file_paths": self.ctx_list,
         }
+        self._bulk_file_methods = {
+            "size": self.fctx_size,
+            "data": self.fctx_node_data,
+            "flags": self.fctx_flags,
+            "is_binary": self.is_binary,
+            "md5": self.md5_hash,
+        }
 
     def _get_ctx(self, repo, ref):
         return get_ctx(repo, ref)
@@ -194,7 +238,7 b' class HgRemote(RemoteBase):'
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
-        return util.version()
+        return safe_str(util.version())
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
@@ -210,10 +254,11 b' class HgRemote(RemoteBase):'
     def bookmarks(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bookmarks(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
-            return dict(repo._bookmarks)
+            return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo._bookmarks.items()}
 
         return _bookmarks(context_uid, repo_id)
 
@@ -221,16 +266,17 b' class HgRemote(RemoteBase):'
     def branches(self, wire, normal, closed):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _branches(_context_uid, _repo_id, _normal, _closed):
             repo = self._factory.repo(wire)
             iter_branches = repo.branchmap().iterbranches()
             bt = {}
-            for branch_name, _heads, tip, is_closed in iter_branches:
+            for branch_name, _heads, tip_node, is_closed in iter_branches:
                 if normal and not is_closed:
-                    bt[branch_name] = tip
+                    bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
                 if closed and is_closed:
-                    bt[branch_name] = tip
+                    bt[safe_str(branch_name)] = ascii_str(hex(tip_node))
 
             return bt
 
@@ -240,12 +286,14 b' class HgRemote(RemoteBase):'
     def bulk_request(self, wire, commit_id, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _commit_id, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
                     result[attr] = method(wire, commit_id)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(
@@ -258,6 +306,7 b' class HgRemote(RemoteBase):'
     def ctx_branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_branch(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -269,6 +318,7 b' class HgRemote(RemoteBase):'
     def ctx_date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_date(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -286,6 +336,7 b' class HgRemote(RemoteBase):'
     def ctx_files(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_files(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -304,6 +355,7 b' class HgRemote(RemoteBase):'
     def ctx_parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_parents(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -317,6 +369,7 b' class HgRemote(RemoteBase):'
     def ctx_children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_children(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -330,6 +383,7 b' class HgRemote(RemoteBase):'
     def ctx_phase(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_phase(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -342,6 +396,7 b' class HgRemote(RemoteBase):'
     def ctx_obsolete(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -353,6 +408,7 b' class HgRemote(RemoteBase):'
     def ctx_hidden(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_hidden(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
@@ -384,46 +440,42 b' class HgRemote(RemoteBase):'
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
-        _proto = None
-        if '+' in url[:url.find('://')]:
-            _proto = url[0:url.find('+')]
-            url = url[url.find('+') + 1:]
+        url, _proto = normalize_url_for_hg(url)
+        url_obj = url_parser(safe_bytes(url))
+
+        test_uri = safe_str(url_obj.authinfo()[0])
+        authinfo = url_obj.authinfo()[1]
+        obfuscated_uri = get_obfuscated_url(url_obj)
+        log.info("Checking URL for remote cloning/import: %s", obfuscated_uri)
+
         handlers = []
-        url_obj = url_parser(url)
-        test_uri, authinfo = url_obj.authinfo()
-        url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
-        url_obj.query = obfuscate_qs(url_obj.query)
-
-        cleaned_uri = str(url_obj)
-        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
-
         if authinfo:
             # create a password manager
-            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
-        o = urllib2.build_opener(*handlers)
+        o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
-        q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
-        qs = '?%s' % urllib.urlencode(q)
-        cu = "%s%s" % (test_uri, qs)
-        req = urllib2.Request(cu, None, {})
+        q.update({'pairs': "{}-{}".format('0' * 40, '0' * 40)})
+        qs = '?%s' % urllib.parse.urlencode(q)
+        cu = f"{test_uri}{qs}"
+        req = urllib.request.Request(cu, None, {})
 
         try:
-            log.debug("Trying to open URL %s", cleaned_uri)
+            log.debug("Trying to open URL %s", obfuscated_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
-            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
+            log.warning("URL cannot be opened: %s", obfuscated_uri, exc_info=True)
             # means it cannot be cloned
-            raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
+            raise exceptions.URLError(e)(f"[{obfuscated_uri}] org_exc: {e}")
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
@@ -432,19 +484,18 b' class HgRemote(RemoteBase):'
             else:
                 # check for pure hg repos
                 log.debug(
-                    "Verifying if URL is a Mercurial repository: %s",
-                    cleaned_uri)
+                    "Verifying if URL is a Mercurial repository: %s", obfuscated_uri)
                 ui = make_ui_from_config(config)
-                peer_checker = makepeer(ui, url)
-                peer_checker.lookup('tip')
+                peer_checker = makepeer(ui, safe_bytes(url))
+                peer_checker.lookup(b'tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
-                        cleaned_uri)
+                        obfuscated_uri)
             raise exceptions.URLError(e)(
                 "url [%s] does not look like an hg repo org_exc: %s"
-                % (cleaned_uri, e))
+                % (obfuscated_uri, e))
 
-        log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
+        log.info("URL is a valid Mercurial repository: %s", obfuscated_uri)
         return True
 
     @reraise_safe_exceptions
@@ -452,14 +503,17 b' class HgRemote(RemoteBase):'
         repo = self._factory.repo(wire)
 
         if file_filter:
-            match_filter = match(file_filter[0], '', [file_filter[1]])
+            # unpack the file-filter
+            repo_path, node_path = file_filter
+            match_filter = match(safe_bytes(repo_path), b'', [safe_bytes(node_path)])
         else:
             match_filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
 
         try:
-            return "".join(patch.diff(
-                repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts))
+            diff_iter = patch.diff(
+                repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts)
+            return BytesEnvelope(b"".join(diff_iter))
         except RepoLookupError as e:
             raise exceptions.LookupException(e)()
 
@@ -467,12 +521,13 b' class HgRemote(RemoteBase):'
     def node_history(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
             repo = self._factory.repo(wire)
 
             ctx = self._get_ctx(repo, revision)
-            fctx = ctx.filectx(path)
+            fctx = ctx.filectx(safe_bytes(path))
 
             def history_iter():
                 limit_rev = fctx.rev()
@@ -498,11 +553,12 b' class HgRemote(RemoteBase):'
     def node_history_untill(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history_until(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
-            fctx = ctx.filectx(path)
+            fctx = ctx.filectx(safe_bytes(path))
 
             file_log = list(fctx.filelog())
             if limit:
@@ -513,10 +569,29 b' class HgRemote(RemoteBase):'
         return _node_history_until(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
+
+    @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
+        fctx = ctx.filectx(safe_bytes(path))
 
         result = []
         for i, annotate_obj in enumerate(fctx.annotate(), 1):
@@ -530,18 +605,19 b' class HgRemote(RemoteBase):'
     def fctx_node_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
-        return fctx.data()
+        fctx = ctx.filectx(safe_bytes(path))
+        return BytesEnvelope(fctx.data())
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_flags(_repo_id, _commit_id, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
-            fctx = ctx.filectx(path)
+            fctx = ctx.filectx(safe_bytes(path))
             return fctx.flags()
 
         return _fctx_flags(repo_id, commit_id, path)
@@ -550,11 +626,12 b' class HgRemote(RemoteBase):'
     def fctx_size(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_size(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
-            fctx = ctx.filectx(path)
+            fctx = ctx.filectx(safe_bytes(path))
             return fctx.size()
         return _fctx_size(repo_id, commit_id, path)
 
@@ -562,44 +639,59 b' class HgRemote(RemoteBase):'
     def get_all_commit_ids(self, wire, name):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id, _name):
             repo = self._factory.repo(wire)
-            repo = repo.filtered(name)
-            revs = map(lambda x: hex(x[7]), repo.changelog.index)
+            revs = [ascii_str(repo[x].hex()) for x in repo.filtered(b'visible').changelog.revs()]
             return revs
         return _get_all_commit_ids(context_uid, repo_id, name)
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
-        return repo.ui.config(section, name, untrusted=untrusted)
+        return repo.ui.config(ascii_bytes(section), ascii_bytes(name), untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
-            return largefiles.lfutil.isstandin(path)
+            return largefiles.lfutil.isstandin(safe_bytes(path))
 
         return _is_large_file(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, revision, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _sha, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
-            fctx = ctx.filectx(path)
+            fctx = ctx.filectx(safe_bytes(path))
             return fctx.isbinary()
 
         return _is_binary(repo_id, revision, path)
 
     @reraise_safe_exceptions
+    def md5_hash(self, wire, revision, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _md5_hash(_repo_id, _sha, _path):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, revision)
+            fctx = ctx.filectx(safe_bytes(path))
+            return hashlib.md5(fctx.data()).hexdigest()
+
+        return _md5_hash(repo_id, revision, path)
+
+    @reraise_safe_exceptions
     def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
@@ -627,11 +719,10 b' class HgRemote(RemoteBase):'
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _lookup(_context_uid, _repo_id, _revision, _both):
-
             repo = self._factory.repo(wire)
             rev = _revision
             if isinstance(rev, int):
@@ -644,7 +735,7 b' class HgRemote(RemoteBase):'
                     rev = rev + -1
             try:
                 ctx = self._get_ctx(repo, rev)
-            except (TypeError, RepoLookupError) as e:
+            except (TypeError, RepoLookupError, binascii.Error) as e:
                 e._org_exc_tb = traceback.format_exc()
                 raise exceptions.LookupException(e)(rev)
             except LookupError as e:
@@ -667,12 +758,12 b' class HgRemote(RemoteBase):'
         repo = self._factory.repo(wire)
 
         # Disable any prompts for this repo
-        repo.ui.setconfig('ui', 'interactive', 'off', '-y')
+        repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
-        bookmarks = dict(repo._bookmarks).keys()
-        remote = peer(repo, {}, url)
+        bookmarks = list(dict(repo._bookmarks).keys())
+        remote = peer(repo, {}, safe_bytes(url))
         # Disable any prompts for this remote
-        remote.ui.setconfig('ui', 'interactive', 'off', '-y')
+        remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         return exchange.push(
             repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
@@ -686,12 +777,15 b' class HgRemote(RemoteBase):'
     @reraise_safe_exceptions
     def rev_range(self, wire, commit_filter):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _rev_range(_context_uid, _repo_id, _filter):
             repo = self._factory.repo(wire)
-            revisions = [rev for rev in revrange(repo, commit_filter)]
+            revisions = [
+                ascii_str(repo[rev].hex())
+                for rev in revrange(repo, list(map(ascii_bytes, commit_filter)))
+            ]
             return revisions
 
         return _rev_range(context_uid, repo_id, sorted(commit_filter))
@@ -710,17 +804,18 b' class HgRemote(RemoteBase):'
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
-        revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
+        revs = [ascii_str(repo[r].hex()) for r in range(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
-        other_path = kwargs.pop('other_path', None)
+        org_path = safe_bytes(wire["path"])
+        other_path = safe_bytes(kwargs.pop('other_path', ''))
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
-            repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
+            repo = unionrepo.makeunionrepository(baseui, other_path, org_path)
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
@@ -764,17 +859,20 b' class HgRemote(RemoteBase):'
     def tags(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tags(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
-            return repo.tags()
+            return {safe_str(name): ascii_str(hex(sha)) for name, sha in repo.tags().items()}
 
         return _tags(context_uid, repo_id)
 
     @reraise_safe_exceptions
-    def update(self, wire, node=None, clean=False):
+    def update(self, wire, node='', clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
+        node = safe_bytes(node)
+
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
@@ -800,10 +898,10 b' class HgRemote(RemoteBase):'
 
         baseui.write = write
         if branch:
-            args = [branch]
+            args = [safe_bytes(branch)]
         else:
             args = []
-        commands.heads(baseui, repo, template='{node} ', *args)
+        commands.heads(baseui, repo, template=b'{node} ', *args)
 
         return output.getvalue()
 
@@ -812,63 +910,61 b' class HgRemote(RemoteBase):'
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
-        a = changelog.ancestor(lookup(revision1), lookup(revision2))
+        a = changelog.ancestor(lookup(safe_bytes(revision1)), lookup(safe_bytes(revision2)))
         return hex(a)
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
-        clone(baseui, source, dest, noupdate=not update_after_clone)
+        clone(baseui, safe_bytes(source), safe_bytes(dest), noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
 
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        publishing = baseui.configbool('phases', 'publish')
-        if publishing:
-            new_commit = 'public'
-        else:
-            new_commit = 'draft'
+        publishing = baseui.configbool(b'phases', b'publish')
 
-        def _filectxfn(_repo, ctx, path):
+        def _filectxfn(_repo, ctx, path: bytes):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
-            if path in removed:
+            if safe_str(path) in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
-                if node['path'] == path:
+                if safe_bytes(node['path']) == path:
                     return memfilectx(
                         _repo,
                         changectx=ctx,
-                        path=node['path'],
-                        data=node['content'],
+                        path=safe_bytes(node['path']),
+                        data=safe_bytes(node['content']),
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copysource=False)
+            abort_exc = exceptions.AbortException()
+            raise abort_exc(f"Given path haven't been marked as added, changed or removed ({path})")
 
-            raise exceptions.AbortException()(
-                "Given path haven't been marked as added, "
-                "changed or removed (%s)" % path)
-
-        with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
-
+        if publishing:
+            new_commit_phase = b'public'
+        else:
+            new_commit_phase = b'draft'
+        with repo.ui.configoverride({(b'phases', b'new-commit'): new_commit_phase}):
+            kwargs = {safe_bytes(k): safe_bytes(v) for k, v in extra.items()}
             commit_ctx = memctx(
                 repo=repo,
                 parents=parents,
-                text=message,
-                files=files,
+                text=safe_bytes(message),
+                files=[safe_bytes(x) for x in files],
                 filectxfn=_filectxfn,
-                user=user,
+                user=safe_bytes(user),
                 date=(commit_time, commit_timezone),
-                extra=extra)
+                extra=kwargs)
 
             n = repo.commitctx(commit_ctx)
             new_id = hex(n)
@@ -879,11 +975,11 b' class HgRemote(RemoteBase):'
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         # Disable any prompts for this repo
-        repo.ui.setconfig('ui', 'interactive', 'off', '-y')
+        repo.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
-        remote = peer(repo, {}, url)
+        remote = peer(repo, {}, safe_bytes(url))
         # Disable any prompts for this remote
-        remote.ui.setconfig('ui', 'interactive', 'off', '-y')
+        remote.ui.setconfig(b'ui', b'interactive', b'off', b'-y')
 
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
@@ -892,34 +988,47 b' class HgRemote(RemoteBase):'
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
-    def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
+    def pull_cmd(self, wire, source, bookmark='', branch='', revision='', hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
+        source = safe_bytes(source)
+
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
+
         if bookmark:
-            opts['bookmark'] = bookmark
+            opts['bookmark'] = [safe_bytes(x) for x in bookmark] \
+                if isinstance(bookmark, list) else safe_bytes(bookmark)
+
         if branch:
-            opts['branch'] = branch
+            opts['branch'] = [safe_bytes(x) for x in branch] \
+                if isinstance(branch, list) else safe_bytes(branch)
+
         if revision:
-            opts['rev'] = revision
+            opts['rev'] = [safe_bytes(x) for x in revision] \
+                if isinstance(revision, list) else safe_bytes(revision)
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
-    def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
+    def push(self, wire, revisions, dest_path, hooks: bool = True, push_branches: bool = False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
-        commands.push(baseui, repo, dest=dest_path, rev=revisions,
+
+        revisions = [safe_bytes(x) for x in revisions] \
+            if isinstance(revisions, list) else safe_bytes(revisions)
+
+        commands.push(baseui, repo, safe_bytes(dest_path),
+                      rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
-        hgext_strip(
+        hgext_strip.strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
@@ -943,25 +1052,25 b' class HgRemote(RemoteBase):'
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        repo.ui.setconfig('ui', 'merge', 'internal:dump')
+        repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
-        repo.ui.setconfig('ui', 'interactive', False)
-        commands.merge(baseui, repo, rev=revision)
+        repo.ui.setconfig(b'ui', b'interactive', False)
+        commands.merge(baseui, repo, rev=safe_bytes(revision))
 
     @reraise_safe_exceptions
     def merge_state(self, wire):
         repo = self._factory.repo(wire)
-        repo.ui.setconfig('ui', 'merge', 'internal:dump')
+        repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
-        repo.ui.setconfig('ui', 'interactive', False)
+        repo.ui.setconfig(b'ui', b'interactive', False)
         ms = hg_merge.mergestate(repo)
         return [x for x in ms.unresolved()]
 
@@ -969,20 +1078,22 b' class HgRemote(RemoteBase):'
     def commit(self, wire, message, username, close_branch=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        repo.ui.setconfig('ui', 'username', username)
-        commands.commit(baseui, repo, message=message, close_branch=close_branch)
+        repo.ui.setconfig(b'ui', b'username', safe_bytes(username))
+        commands.commit(baseui, repo, message=safe_bytes(message), close_branch=close_branch)
 
     @reraise_safe_exceptions
-    def rebase(self, wire, source=None, dest=None, abort=False):
+    def rebase(self, wire, source='', dest='', abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        repo.ui.setconfig('ui', 'merge', 'internal:dump')
+        repo.ui.setconfig(b'ui', b'merge', b'internal:dump')
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
-        repo.ui.setconfig('ui', 'interactive', False)
-        rebase.rebase(baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
+        repo.ui.setconfig(b'ui', b'interactive', False)
+
+        rebase.rebase(baseui, repo, base=safe_bytes(source or ''), dest=safe_bytes(dest or ''),
+                      abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
@@ -992,17 +1103,18 b' class HgRemote(RemoteBase):'
 
         date = (tag_time, tag_timezone)
         try:
-            hg_tag.tag(repo, name, node, message, local, user, date)
+            hg_tag.tag(repo, safe_bytes(name), node, safe_bytes(message), local, safe_bytes(user), date)
         except Abort as e:
             log.exception("Tag operation aborted")
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
-    def bookmark(self, wire, bookmark, revision=None):
+    def bookmark(self, wire, bookmark, revision=''):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
+        revision = revision or ''
+        commands.bookmark(baseui, repo, safe_bytes(bookmark), rev=safe_bytes(revision), force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
@@ -1021,8 +1133,8 b' class HgRemote(RemoteBase):'
         pass
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def file_walker(_commit_id, path):
             repo = self._factory.repo(wire)
@@ -1042,6 +1154,6 b' class HgRemote(RemoteBase):'
 
                 yield ArchiveNode(file_path, mode, is_link, ctx[fn].data)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
 

vcsserver/remote/svn.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,32 +15,34 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
-from __future__ import absolute_import
 
 import os
 import subprocess
-import time
-from urllib2 import URLError
-import urlparse
+from urllib.error import URLError
+import urllib.parse
 import logging
 import posixpath as vcspath
-import StringIO
-import urllib
+import io
+import urllib.request
+import urllib.parse
+import urllib.error
 import traceback
 
-import svn.client
-import svn.core
-import svn.delta
-import svn.diff
-import svn.fs
-import svn.repos
+
+import svn.client  # noqa
+import svn.core  # noqa
+import svn.delta  # noqa
+import svn.diff  # noqa
+import svn.fs  # noqa
+import svn.repos  # noqa
 
 from vcsserver import svn_diff, exceptions, subprocessio, settings
-from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
+from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope
 from vcsserver.exceptions import NoContentException
-from vcsserver.utils import safe_str
+from vcsserver.str_utils import safe_str, safe_bytes
+from vcsserver.type_utils import assert_bytes
 from vcsserver.vcs_base import RemoteBase
-
+from vcsserver.lib.svnremoterepo import svnremoterepo
 log = logging.getLogger(__name__)
 
 
@@ -52,7 +54,7 b' svn_compatible_versions_map = {'
     'pre-1.9-compatible': '1.8',
 }
 
-current_compatible_version = '1.12'
+current_compatible_version = '1.14'
 
 
 def reraise_safe_exceptions(func):
@@ -63,7 +65,7 b' def reraise_safe_exceptions(func):'
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in svn remote call")
-                raise_from_original(exceptions.UnhandledException(e))
+                raise_from_original(exceptions.UnhandledException(e), e)
             raise
     return wrapper
 
@@ -82,12 +84,12 b' class SubversionFactory(RepoFactory):'
                     or compatible_version
                 fs_config['compatible-version'] = compatible_version_string
 
-            log.debug('Create SVN repo with config "%s"', fs_config)
+            log.debug('Create SVN repo with config `%s`', fs_config)
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
 
-        log.debug('Got SVN object: %s', repo)
+        log.debug('repository created: got SVN object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
@@ -107,9 +109,39 b' class SvnRemote(RemoteBase):'
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
-        # TODO: Remove once we do not use internal Mercurial objects anymore
-        # for subversion
-        self._hg_factory = hg_factory
+
+        self._bulk_methods = {
+            # NOT supported in SVN ATM...
+        }
+        self._bulk_file_methods = {
+            "size": self.get_file_size,
+            "data": self.get_file_content,
+            "flags": self.get_node_type,
+            "is_binary": self.is_binary,
+            "md5": self.md5_hash
+        }
+
+    @reraise_safe_exceptions
+    def bulk_file_request(self, wire, commit_id, path, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        # since we use unified API, we need to cast from str to in for SVN
+        commit_id = int(commit_id)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_file_request(_repo_id, _commit_id, _path, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_file_methods[attr]
+                    wire.update({'cache': False})  # disable cache for bulk calls so we don't double cache
+                    result[attr] = method(wire, _commit_id, _path)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(f'Unknown bulk attribute: "{attr}"')
+            return BinaryEnvelope(result)
+
+        return _bulk_file_request(repo_id, commit_id, path, sorted(pre_load))
 
     @reraise_safe_exceptions
     def discover_svn_version(self):
@@ -118,31 +150,27 b' class SvnRemote(RemoteBase):'
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
-        return svn_ver
+        return safe_str(svn_ver)
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
-
         try:
             return self.lookup(wire, -1) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
-    def check_url(self, url, config_items):
-        # this can throw exception if not installed, but we detect this
-        from hgsubversion import svnrepo
+    def check_url(self, url, config):
 
-        baseui = self._hg_factory._create_config(config_items)
-        # uuid function get's only valid UUID from proper repo, else
+        # uuid function gets only valid UUID from proper repo, else
         # throws exception
+        username, password, src_url = self.get_url_and_credentials(url)
         try:
-            svnrepo.svnremoterepo(baseui, url).svn.uuid
+            svnremoterepo(safe_bytes(username), safe_bytes(password), safe_bytes(src_url)).svn().uuid
         except Exception:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
-            raise URLError(
-                '"%s" is not a valid Subversion source url.' % (url, ))
+            raise URLError(f'"{url}" is not a valid Subversion source url.')
         return True
 
     def is_path_valid_repository(self, wire, path):
@@ -173,6 +201,7 b' class SvnRemote(RemoteBase):'
         stdout, stderr = subprocessio.run_command(cmd)
         return stdout
 
+    @reraise_safe_exceptions
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
@@ -181,6 +210,7 b' class SvnRemote(RemoteBase):'
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
+    @reraise_safe_exceptions
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
@@ -198,10 +228,12 b' class SvnRemote(RemoteBase):'
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
+    @reraise_safe_exceptions
     def revision_properties(self, wire, revision):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _revision_properties(_repo_id, _revision):
             repo = self._factory.repo(wire)
@@ -228,7 +260,7 b' class SvnRemote(RemoteBase):'
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
-        for path, change in editor.changes.iteritems():
+        for path, change in editor.changes.items():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
@@ -243,7 +275,7 b' class SvnRemote(RemoteBase):'
                 removed.append(path)
             else:
                 raise NotImplementedError(
-                    "Action %s not supported on path %s" % (
+                    "Action {} not supported on path {}".format(
                         change.action, path))
 
         changes = {
@@ -257,6 +289,7 b' class SvnRemote(RemoteBase):'
     def node_history(self, wire, path, revision, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
             cross_copies = False
@@ -276,9 +309,11 b' class SvnRemote(RemoteBase):'
             return history_revisions
         return _assert_correct_path(context_uid, repo_id, path, revision, limit)
 
+    @reraise_safe_exceptions
     def node_properties(self, wire, path, revision):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_properties(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
@@ -288,7 +323,7 b' class SvnRemote(RemoteBase):'
         return _node_properties(repo_id, path, revision)
 
     def file_annotate(self, wire, path, revision):
-        abs_path = 'file://' + urllib.pathname2url(
+        abs_path = 'file://' + urllib.request.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
@@ -314,12 +349,14 b' class SvnRemote(RemoteBase):'
 
         return annotations
 
-    def get_node_type(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_node_type(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _get_node_type(_repo_id, _path, _revision):
+        def _get_node_type(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             fs_ptr = svn.repos.fs(repo)
             if _revision is None:
@@ -327,12 +364,14 b' class SvnRemote(RemoteBase):'
             root = svn.fs.revision_root(fs_ptr, _revision)
             node = svn.fs.check_path(root, path)
             return NODE_TYPE_MAPPING.get(node, None)
-        return _get_node_type(repo_id, path, revision)
+        return _get_node_type(repo_id, revision, path)
 
-    def get_nodes(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_nodes(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_nodes(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
@@ -342,27 +381,32 b' class SvnRemote(RemoteBase):'
             root = svn.fs.revision_root(fsobj, _revision)
             entries = svn.fs.dir_entries(root, path)
             result = []
-            for entry_path, entry_info in entries.iteritems():
+            for entry_path, entry_info in entries.items():
                 result.append(
                     (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
             return result
         return _get_nodes(repo_id, path, revision)
 
-    def get_file_content(self, wire, path, rev=None):
+    @reraise_safe_exceptions
+    def get_file_content(self, wire, rev=None, path=''):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
+
         if rev is None:
-            rev = svn.fs.youngest_revision(fsobj)
+            rev = svn.fs.youngest_rev(fsobj)
+
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
-        return content.read()
+        return BytesEnvelope(content.read())
 
-    def get_file_size(self, wire, path, revision=None):
+    @reraise_safe_exceptions
+    def get_file_size(self, wire, revision=None, path=''):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
-        def _get_file_size(_repo_id, _path, _revision):
+        def _get_file_size(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             if _revision is None:
@@ -370,17 +414,17 b' class SvnRemote(RemoteBase):'
             root = svn.fs.revision_root(fsobj, _revision)
             size = svn.fs.file_length(root, path)
             return size
-        return _get_file_size(repo_id, path, revision)
+        return _get_file_size(repo_id, revision, path)
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
-    def get_url_and_credentials(self, src_url):
-        obj = urlparse.urlparse(src_url)
-        username = obj.username or None
-        password = obj.password or None
+    def get_url_and_credentials(self, src_url) -> tuple[str, str, str]:
+        obj = urllib.parse.urlparse(src_url)
+        username = obj.username or ''
+        password = obj.password or ''
         return username, password, src_url
 
     def import_remote_repository(self, wire, src_url):
@@ -411,16 +455,17 b' class SvnRemote(RemoteBase):'
         log.debug('Return process ended with code: %s', rdump.returncode)
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
-            log.error('svnrdump dump failed: statuscode %s: message: %s',
-                      rdump.returncode, errors)
+            log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
+
             reason = 'UNKNOWN'
-            if 'svnrdump: E230001:' in errors:
+            if b'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
 
             if reason == 'UNKNOWN':
-                reason = 'UNKNOWN:{}'.format(errors)
+                reason = f'UNKNOWN:{safe_str(errors)}'
+
             raise Exception(
-                'Failed to dump the remote repository from %s. Reason:%s' % (
+                'Failed to dump the remote repository from {}. Reason:{}'.format(
                     src_url, reason))
         if load.returncode != 0:
             raise Exception(
@@ -428,8 +473,9 b' class SvnRemote(RemoteBase):'
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
-        assert isinstance(message, str)
-        assert isinstance(author, str)
+
+        message = safe_bytes(message)
+        author = safe_bytes(author)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
@@ -453,6 +499,7 b' class SvnRemote(RemoteBase):'
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
+    @reraise_safe_exceptions
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
@@ -461,12 +508,12 b' class SvnRemote(RemoteBase):'
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
-            return diff_creator.generate_diff()
+            return BytesEnvelope(diff_creator.generate_diff())
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
-                "Path might not exist %s, %s" % (path1, path2))
-            return ""
+                "Path might not exist %s, %s", path1, path2)
+            return BytesEnvelope(b'')
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
@@ -475,16 +522,29 b' class SvnRemote(RemoteBase):'
     @reraise_safe_exceptions
     def is_binary(self, wire, rev, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
 
-        region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _rev, _path):
-            raw_bytes = self.get_file_content(wire, path, rev)
-            return raw_bytes and '\0' in raw_bytes
+            raw_bytes = self.get_file_content(wire, rev, path)
+            if not raw_bytes:
+                return False
+            return b'\0' in raw_bytes
 
         return _is_binary(repo_id, rev, path)
 
     @reraise_safe_exceptions
+    def md5_hash(self, wire, rev, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        region = self._region(wire)
+
+        @region.conditional_cache_on_arguments(condition=cache_on)
+        def _md5_hash(_repo_id, _rev, _path):
+            return ''
+
+        return _md5_hash(repo_id, rev, path)
+
+    @reraise_safe_exceptions
     def run_svn_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
@@ -500,14 +560,14 b' class SvnRemote(RemoteBase):'
 
         try:
             _opts.update(opts)
-            p = subprocessio.SubprocessIOChunker(cmd, **_opts)
+            proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
-            return ''.join(p), ''.join(p.error)
-        except (EnvironmentError, OSError) as err:
+            return b''.join(proc), b''.join(proc.stderr)
+        except OSError as err:
             if safe_call:
                 return '', safe_str(err).strip()
             else:
-                cmd = ' '.join(cmd)  # human friendly CMD
+                cmd = ' '.join(map(safe_str, cmd))  # human friendly CMD
                 tb_err = ("Couldn't run svn command (%s).\n"
                           "Original error was:%s\n"
                           "Call options:%s\n"
@@ -522,9 +582,8 b' class SvnRemote(RemoteBase):'
         binary_dir = settings.BINARY_DIR
         executable = None
         if binary_dir:
-            executable = os.path.join(binary_dir, 'python')
-        return install_svn_hooks(
-            repo_path, executable=executable, force_create=force)
+            executable = os.path.join(binary_dir, 'python3')
+        return install_svn_hooks(repo_path, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
@@ -541,13 +600,14 b' class SvnRemote(RemoteBase):'
         pass
 
     @reraise_safe_exceptions
-    def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
-                     archive_dir_name, commit_id):
+    def archive_repo(self, wire, archive_name_key, kind, mtime, archive_at_path,
+                     archive_dir_name, commit_id, cache_config):
 
         def walk_tree(root, root_dir, _commit_id):
             """
             Special recursive svn repo walker
             """
+            root_dir = safe_bytes(root_dir)
 
             filemode_default = 0o100644
             filemode_executable = 0o100755
@@ -560,10 +620,10 b' class SvnRemote(RemoteBase):'
                     # return only DIR, and then all entries in that dir
                     yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
                     new_root = os.path.join(root_dir, f_name)
-                    for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
-                        yield _f_name, _f_data, _f_type
+                    yield from walk_tree(root, new_root, _commit_id)
                 else:
-                    f_path = os.path.join(root_dir, f_name).rstrip('/')
+
+                    f_path = os.path.join(root_dir, f_name).rstrip(b'/')
                     prop_list = svn.fs.node_proplist(root, f_path)
 
                     f_mode = filemode_default
@@ -601,8 +661,8 b' class SvnRemote(RemoteBase):'
                     data_stream = f_data['content_stream']
                     yield ArchiveNode(file_path, mode, is_link, data_stream)
 
-        return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
-                            archive_dir_name, commit_id)
+        return store_archive_in_cache(
+            file_walker, archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config=cache_config)
 
 
 class SvnDiffer(object):
@@ -643,15 +703,15 b' class SvnDiffer(object):'
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
-    def generate_diff(self):
-        buf = StringIO.StringIO()
+    def generate_diff(self) -> bytes:
+        buf = io.BytesIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
-    def _generate_dir_diff(self, buf):
+    def _generate_dir_diff(self, buf: io.BytesIO):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
@@ -672,7 +732,7 b' class SvnDiffer(object):'
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
-    def _generate_file_diff(self, buf):
+    def _generate_file_diff(self, buf: io.BytesIO):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
@@ -684,7 +744,13 b' class SvnDiffer(object):'
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
-            self, buf, change, tgt_path, tgt_base, src_path, src_base):
+            self, buf: io.BytesIO, change, tgt_path, tgt_base, src_path, src_base):
+
+        tgt_path_bytes = safe_bytes(tgt_path)
+        tgt_path = safe_str(tgt_path)
+
+        src_path_bytes = safe_bytes(src_path)
+        src_path = safe_str(src_path)
 
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
@@ -697,55 +763,55 b' class SvnDiffer(object):'
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
-        if mime_type and not mime_type.startswith('text'):
+        if mime_type and not mime_type.startswith(b'text'):
             self.binary_content = True
-            buf.write("=" * 67 + '\n')
-            buf.write("Cannot display: file marked as a binary type.\n")
-            buf.write("svn:mime-type = %s\n" % mime_type)
-        buf.write("Index: %s\n" % (tgt_path, ))
-        buf.write("=" * 67 + '\n')
-        buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
-            'tgt_path': tgt_path})
+            buf.write(b"=" * 67 + b'\n')
+            buf.write(b"Cannot display: file marked as a binary type.\n")
+            buf.write(b"svn:mime-type = %s\n" % mime_type)
+        buf.write(b"Index: %b\n" % tgt_path_bytes)
+        buf.write(b"=" * 67 + b'\n')
+        buf.write(b"diff --git a/%b b/%b\n" % (tgt_path_bytes, tgt_path_bytes))
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
-            buf.write("new file mode 10644\n")
+            buf.write(b"new file mode 10644\n")
+
+            # TODO(marcink): intro to binary detection of svn patches
+            # if self.binary_content:
+            #     buf.write(b'GIT binary patch\n')
 
-            #TODO(marcink): intro to binary detection of svn patches
+            buf.write(b"--- /dev/null\t(revision 0)\n")
+            src_lines = []
+        else:
+            if change == 'delete':
+                buf.write(b"deleted file mode 10644\n")
+
+            # TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
-            buf.write("--- /dev/null\t(revision 0)\n")
-            src_lines = []
-        else:
-            if change == 'delete':
-                buf.write("deleted file mode 10644\n")
-
-            #TODO(marcink): intro to binary detection of svn patches
-            # if self.binary_content:
-            #     buf.write('GIT binary patch\n')
-
-            buf.write("--- a/%s\t(revision %s)\n" % (
-                src_path, self.src_rev))
+            buf.write(b"--- a/%b\t(revision %d)\n" % (src_path_bytes, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
-            buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
+            buf.write(b"+++ /dev/null\t(revision %d)\n" % self.tgt_rev)
             tgt_lines = []
         else:
-            buf.write("+++ b/%s\t(revision %s)\n" % (
-                tgt_path, self.tgt_rev))
+            buf.write(b"+++ b/%b\t(revision %d)\n" % (tgt_path_bytes, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
+        # we made our diff header, time to generate the diff content into our buffer
+
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
+
             buf.writelines(udiff)
 
-    def _get_mime_type(self, path):
+    def _get_mime_type(self, path) -> bytes:
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
@@ -761,7 +827,9 b' class SvnDiffer(object):'
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
-        content = svn.core.Stream(svn.fs.file_contents(fs_root, node_path)).read()
+        content = svn.core.Stream(
+            svn.fs.file_contents(fs_root, node_path)).read()
+
         return content.splitlines(True)
 
 
@@ -799,7 +867,7 b' class TxnNodeProcessor(object):'
     """
 
     def __init__(self, node, txn_root):
-        assert isinstance(node['path'], str)
+        assert_bytes(node['path'])
 
         self.node = node
         self.txn_root = txn_root
@@ -835,23 +903,24 b' class TxnNodeProcessor(object):'
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
-        assert isinstance(self.node['content'], str)
+        assert_bytes(self.node['content'])
+
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
-        for key, value in properties.iteritems():
+        for key, value in properties.items():
             svn.fs.change_node_prop(
-                self.txn_root, self.node['path'], key, value)
+                self.txn_root, self.node['path'], safe_bytes(key), safe_bytes(value))
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
-    return timestamp * 1E6
+    return int(timestamp * 1E6)
 
 
 def svn_opt_revision_value_t(num):

vcsserver/remote_wsgi.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/scm_app.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,7 +27,7 b' import mercurial.hgweb.hgweb_mod'
 import webob.exc
 
 from vcsserver import pygrack, exceptions, settings, git_lfs
-
+from vcsserver.str_utils import ascii_bytes, safe_bytes
 
 log = logging.getLogger(__name__)
 
@@ -81,7 +81,7 b' class HgWeb(mercurial.hgweb.hgweb_mod.hg'
         first_chunk = None
 
         try:
-            data = gen.next()
+            data = next(gen)
 
             def first_chunk():
                 yield data
@@ -94,16 +94,17 b' class HgWeb(mercurial.hgweb.hgweb_mod.hg'
 
     def _runwsgi(self, req, res, repo):
 
-        cmd = req.qsparams.get('cmd', '')
+        cmd = req.qsparams.get(b'cmd', '')
         if not mercurial.wireprotoserver.iscmd(cmd):
             # NOTE(marcink): for unsupported commands, we return bad request
             # internally from HG
+            log.warning('cmd: `%s` is not supported by the mercurial wireprotocol v1', cmd)
             from mercurial.hgweb.common import statusmessage
             res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST)
-            res.setbodybytes('')
+            res.setbodybytes(b'')
             return res.sendresponse()
 
-        return super(HgWeb, self)._runwsgi(req, res, repo)
+        return super()._runwsgi(req, res, repo)
 
 
 def make_hg_ui_from_config(repo_config):
@@ -115,10 +116,13 b' def make_hg_ui_from_config(repo_config):'
     baseui._tcfg = mercurial.config.config()
 
     for section, option, value in repo_config:
-        baseui.setconfig(section, option, value)
+        baseui.setconfig(
+            ascii_bytes(section, allow_bytes=True),
+            ascii_bytes(option, allow_bytes=True),
+            ascii_bytes(value, allow_bytes=True))
 
     # make our hgweb quiet so it doesn't print output
-    baseui.setconfig('ui', 'quiet', 'true')
+    baseui.setconfig(b'ui', b'quiet', b'true')
 
     return baseui
 
@@ -131,11 +135,14 b' def update_hg_ui_from_hgrc(baseui, repo_'
         return
     log.debug('reading hgrc from %s', path)
     cfg = mercurial.config.config()
-    cfg.read(path)
+    cfg.read(ascii_bytes(path))
     for section in HG_UI_SECTIONS:
         for k, v in cfg.items(section):
             log.debug('settings ui from file: [%s] %s=%s', section, k, v)
-            baseui.setconfig(section, k, v)
+            baseui.setconfig(
+                ascii_bytes(section, allow_bytes=True),
+                ascii_bytes(k, allow_bytes=True),
+                ascii_bytes(v, allow_bytes=True))
 
 
 def create_hg_wsgi_app(repo_path, repo_name, config):
@@ -151,7 +158,7 b' def create_hg_wsgi_app(repo_path, repo_n'
     update_hg_ui_from_hgrc(baseui, repo_path)
 
     try:
-        return HgWeb(repo_path, name=repo_name, baseui=baseui)
+        return HgWeb(safe_bytes(repo_path), name=safe_bytes(repo_name), baseui=baseui)
     except mercurial.error.RequirementError as e:
         raise exceptions.RequirementException(e)(e)
 
@@ -225,10 +232,10 b' class GitLFSHandler(object):'
 
 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
     git_path = settings.GIT_EXECUTABLE
-    update_server_info = config.pop('git_update_server_info')
-    git_lfs_enabled = config.pop('git_lfs_enabled')
-    git_lfs_store_path = config.pop('git_lfs_store_path')
-    git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http')
+    update_server_info = config.pop(b'git_update_server_info')
+    git_lfs_enabled = config.pop(b'git_lfs_enabled')
+    git_lfs_store_path = config.pop(b'git_lfs_store_path')
+    git_lfs_http_scheme = config.pop(b'git_lfs_http_scheme', 'http')
     app = GitLFSHandler(
         repo_path, repo_name, git_path, update_server_info, config)
 

vcsserver/server.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/settings.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/subprocessio.py

@@ -23,15 +23,17 b' along with git_http_backend.py Project.'
 If not, see <http://www.gnu.org/licenses/>.
 """
 import os
+import collections
 import logging
-import subprocess32 as subprocess
-from collections import deque
-from threading import Event, Thread
+import subprocess
+import threading
+
+from vcsserver.str_utils import safe_str
 
 log = logging.getLogger(__name__)
 
 
-class StreamFeeder(Thread):
+class StreamFeeder(threading.Thread):
     """
     Normal writing into pipe-like is blocking once the buffer is filled.
     This thread allows a thread to seep data from a file-like into a pipe
@@ -40,24 +42,18 b' class StreamFeeder(Thread):'
     """
 
     def __init__(self, source):
-        super(StreamFeeder, self).__init__()
+        super().__init__()
         self.daemon = True
         filelike = False
-        self.bytes = bytes()
-        if type(source) in (type(''), bytes, bytearray):  # string-like
+        self.bytes = b''
+        if type(source) in (str, bytes, bytearray):  # string-like
             self.bytes = bytes(source)
         else:  # can be either file pointer or file-like
-            if type(source) in (int, long):  # file pointer it is
+            if isinstance(source, int):  # file pointer it is
                 # converting file descriptor (int) stdin into file-like
-                try:
-                    source = os.fdopen(source, 'rb', 16384)
-                except Exception:
-                    pass
+                source = os.fdopen(source, 'rb', 16384)
             # let's see if source is file-like by now
-            try:
-                filelike = source.read
-            except Exception:
-                pass
+            filelike = hasattr(source, 'read')
         if not filelike and not self.bytes:
             raise TypeError("StreamFeeder's source object must be a readable "
                             "file-like, a file descriptor, or a string-like.")
@@ -65,28 +61,31 b' class StreamFeeder(Thread):'
         self.readiface, self.writeiface = os.pipe()
 
     def run(self):
-        t = self.writeiface
+        writer = self.writeiface
         try:
             if self.bytes:
-                os.write(t, self.bytes)
+                os.write(writer, self.bytes)
             else:
                 s = self.source
-                b = s.read(4096)
-                while b:
-                    os.write(t, b)
-                    b = s.read(4096)
+
+                while 1:
+                    _bytes = s.read(4096)
+                    if not _bytes:
+                        break
+                    os.write(writer, _bytes)
+
         finally:
-            os.close(t)
+            os.close(writer)
 
     @property
     def output(self):
         return self.readiface
 
 
-class InputStreamChunker(Thread):
+class InputStreamChunker(threading.Thread):
     def __init__(self, source, target, buffer_size, chunk_size):
 
-        super(InputStreamChunker, self).__init__()
+        super().__init__()
 
         self.daemon = True  # die die die.
 
@@ -95,16 +94,16 b' class InputStreamChunker(Thread):'
         self.chunk_count_max = int(buffer_size / chunk_size) + 1
         self.chunk_size = chunk_size
 
-        self.data_added = Event()
+        self.data_added = threading.Event()
         self.data_added.clear()
 
-        self.keep_reading = Event()
+        self.keep_reading = threading.Event()
         self.keep_reading.set()
 
-        self.EOF = Event()
+        self.EOF = threading.Event()
         self.EOF.clear()
 
-        self.go = Event()
+        self.go = threading.Event()
         self.go.set()
 
     def stop(self):
@@ -115,7 +114,7 b' class InputStreamChunker(Thread):'
             # go of the input because, if successful, .close() will send EOF
             # down the pipe.
             self.source.close()
-        except:
+        except Exception:
             pass
 
     def run(self):
@@ -146,7 +145,7 b' class InputStreamChunker(Thread):'
 
             try:
                 b = s.read(cs)
-            except ValueError:
+            except ValueError:  # probably "I/O operation on closed file"
                 b = ''
 
         self.EOF.set()
@@ -166,18 +165,20 b' class BufferedGenerator(object):'
     StopIteration after the last chunk of data is yielded.
     """
 
-    def __init__(self, source, buffer_size=65536, chunk_size=4096,
+    def __init__(self, name, source, buffer_size=65536, chunk_size=4096,
                  starting_values=None, bottomless=False):
         starting_values = starting_values or []
+        self.name = name
+        self.buffer_size = buffer_size
+        self.chunk_size = chunk_size
 
         if bottomless:
             maxlen = int(buffer_size / chunk_size)
         else:
             maxlen = None
 
-        self.data = deque(starting_values, maxlen)
-        self.worker = InputStreamChunker(source, self.data, buffer_size,
-                                         chunk_size)
+        self.data_queue = collections.deque(starting_values, maxlen)
+        self.worker = InputStreamChunker(source, self.data_queue, buffer_size, chunk_size)
         if starting_values:
             self.worker.data_added.set()
         self.worker.start()
@@ -185,17 +186,21 b' class BufferedGenerator(object):'
     ####################
     # Generator's methods
     ####################
+    def __str__(self):
+        return f'BufferedGenerator(name={self.name} chunk: {self.chunk_size} on buffer: {self.buffer_size})'
 
     def __iter__(self):
         return self
 
-    def next(self):
-        while not len(self.data) and not self.worker.EOF.is_set():
+    def __next__(self):
+
+        while not self.length and not self.worker.EOF.is_set():
             self.worker.data_added.clear()
             self.worker.data_added.wait(0.2)
-        if len(self.data):
+
+        if self.length:
             self.worker.keep_reading.set()
-            return bytes(self.data.popleft())
+            return bytes(self.data_queue.popleft())
         elif self.worker.EOF.is_set():
             raise StopIteration
 
@@ -249,7 +254,7 b' class BufferedGenerator(object):'
     @property
     def done_reading(self):
         """
-        Done_reding does not mean that the iterator's buffer is empty.
+        Done_reading does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
 
@@ -262,31 +267,31 b' class BufferedGenerator(object):'
         """
         returns int.
 
-        This is the lenght of the que of chunks, not the length of
+        This is the length of the queue of chunks, not the length of
         the combined contents in those chunks.
 
         __len__() cannot be meaningfully implemented because this
-        reader is just flying throuh a bottomless pit content and
-        can only know the lenght of what it already saw.
+        reader is just flying through a bottomless pit content and
+        can only know the length of what it already saw.
 
         If __len__() on WSGI server per PEP 3333 returns a value,
-        the responce's length will be set to that. In order not to
+        the response's length will be set to that. In order not to
         confuse WSGI PEP3333 servers, we will not implement __len__
         at all.
         """
-        return len(self.data)
+        return len(self.data_queue)
 
     def prepend(self, x):
-        self.data.appendleft(x)
+        self.data_queue.appendleft(x)
 
     def append(self, x):
-        self.data.append(x)
+        self.data_queue.append(x)
 
     def extend(self, o):
-        self.data.extend(o)
+        self.data_queue.extend(o)
 
     def __getitem__(self, i):
-        return self.data[i]
+        return self.data_queue[i]
 
 
 class SubprocessIOChunker(object):
@@ -314,7 +319,7 b' class SubprocessIOChunker(object):'
 
     - We are multithreaded. Writing in and reading out, err are all sep threads.
     - We support concurrent (in and out) stream processing.
-    - The output is not a stream. It's a queue of read string (bytes, not unicode)
+    - The output is not a stream. It's a queue of read string (bytes, not str)
       chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
     - We are non-blocking in more respects than communicate()
       (reading from subprocess out pauses when internal buffer is full, but
@@ -323,16 +328,16 b' class SubprocessIOChunker(object):'
        does not block the parallel inpipe reading occurring parallel thread.)
 
     The purpose of the object is to allow us to wrap subprocess interactions into
-    and interable that can be passed to a WSGI server as the application's return
+    an iterable that can be passed to a WSGI server as the application's return
     value. Because of stream-processing-ability, WSGI does not have to read ALL
     of the subprocess's output and buffer it, before handing it to WSGI server for
     HTTP response. Instead, the class initializer reads just a bit of the stream
-    to figure out if error ocurred or likely to occur and if not, just hands the
+    to figure out if error occurred or likely to occur and if not, just hands the
     further iteration over subprocess output to the server for completion of HTTP
     response.
 
     The real or perceived subprocess error is trapped and raised as one of
-    EnvironmentError family of exceptions
+    OSError family of exceptions
 
     Example usage:
     #    try:
@@ -342,7 +347,7 b' class SubprocessIOChunker(object):'
     #            buffer_size = 65536,
     #            chunk_size = 4096
     #            )
-    #    except (EnvironmentError) as e:
+    #    except (OSError) as e:
     #        print str(e)
     #        raise e
     #
@@ -358,15 +363,17 b' class SubprocessIOChunker(object):'
     _close_input_fd = None
 
     _closed = False
+    _stdout = None
+    _stderr = None
 
-    def __init__(self, cmd, inputstream=None, buffer_size=65536,
+    def __init__(self, cmd, input_stream=None, buffer_size=65536,
                  chunk_size=4096, starting_values=None, fail_on_stderr=True,
                  fail_on_return_code=True, **kwargs):
         """
         Initializes SubprocessIOChunker
 
         :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
-        :param inputstream: (Default: None) A file-like, string, or file pointer.
+        :param input_stream: (Default: None) A file-like, string, or file pointer.
         :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
         :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
         :param starting_values: (Default: []) An array of strings to put in front of output que.
@@ -376,71 +383,86 b' class SubprocessIOChunker(object):'
                                     exception if the return code is not 0.
         """
 
+        kwargs['shell'] = kwargs.get('shell', True)
+
         starting_values = starting_values or []
-        if inputstream:
-            input_streamer = StreamFeeder(inputstream)
+        if input_stream:
+            input_streamer = StreamFeeder(input_stream)
             input_streamer.start()
-            inputstream = input_streamer.output
-            self._close_input_fd = inputstream
+            input_stream = input_streamer.output
+            self._close_input_fd = input_stream
 
         self._fail_on_stderr = fail_on_stderr
         self._fail_on_return_code = fail_on_return_code
-
-        _shell = kwargs.get('shell', True)
-        kwargs['shell'] = _shell
+        self.cmd = cmd
 
-        _p = subprocess.Popen(cmd, bufsize=-1,
-                              stdin=inputstream,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE,
+        _p = subprocess.Popen(cmd, bufsize=-1, stdin=input_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               **kwargs)
+        self.process = _p
 
-        bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size,
-                                   starting_values)
-        bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)
+        bg_out = BufferedGenerator('stdout', _p.stdout, buffer_size, chunk_size, starting_values)
+        bg_err = BufferedGenerator('stderr', _p.stderr, 10240, 1, bottomless=True)
 
         while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
             # doing this until we reach either end of file, or end of buffer.
-            bg_out.data_added_event.wait(1)
+            bg_out.data_added_event.wait(0.2)
             bg_out.data_added_event.clear()
 
         # at this point it's still ambiguous if we are done reading or just full buffer.
         # Either way, if error (returned by ended process, or implied based on
         # presence of stuff in stderr output) we error out.
         # Else, we are happy.
-        _returncode = _p.poll()
+        return_code = _p.poll()
+        ret_code_ok = return_code in [None, 0]
+        ret_code_fail = return_code is not None and return_code != 0
+        if (
+            (ret_code_fail and fail_on_return_code) or
+            (ret_code_ok and fail_on_stderr and bg_err.length)
+        ):
 
-        if ((_returncode and fail_on_return_code) or
-                (fail_on_stderr and _returncode is None and bg_err.length)):
             try:
                 _p.terminate()
             except Exception:
                 pass
+
             bg_out.stop()
+            out = b''.join(bg_out)
+            self._stdout = out
+
             bg_err.stop()
-            if fail_on_stderr:
-                err = ''.join(bg_err)
-                raise EnvironmentError(
-                    "Subprocess exited due to an error:\n" + err)
-            if _returncode and fail_on_return_code:
-                err = ''.join(bg_err)
+            err = b''.join(bg_err)
+            self._stderr = err
+
+            # code from https://github.com/schacon/grack/pull/7
+            if err.strip() == b'fatal: The remote end hung up unexpectedly' and out.startswith(b'0034shallow '):
+                bg_out = iter([out])
+                _p = None
+            elif err and fail_on_stderr:
+                text_err = err.decode()
+                raise OSError(
+                    f"Subprocess exited due to an error:\n{text_err}")
+
+            if ret_code_fail and fail_on_return_code:
+                text_err = err.decode()
                 if not err:
                     # maybe get empty stderr, try stdout instead
                     # in many cases git reports the errors on stdout too
-                    err = ''.join(bg_out)
-                raise EnvironmentError(
-                    "Subprocess exited with non 0 ret code:%s: stderr:%s" % (
-                        _returncode, err))
+                    text_err = out.decode()
+                raise OSError(
+                    f"Subprocess exited with non 0 ret code:{return_code}: stderr:{text_err}")
 
-        self.process = _p
-        self.output = bg_out
-        self.error = bg_err
-        self.inputstream = inputstream
+        self.stdout = bg_out
+        self.stderr = bg_err
+        self.inputstream = input_stream
+
+    def __str__(self):
+        proc = getattr(self, 'process', 'NO_PROCESS')
+        return f'SubprocessIOChunker: {proc}'
 
     def __iter__(self):
         return self
 
-    def next(self):
+    def __next__(self):
         # Note: mikhail: We need to be sure that we are checking the return
         # code after the stdout stream is closed. Some processes, e.g. git
         # are doing some magic in between closing stdout and terminating the
@@ -449,27 +471,31 b' class SubprocessIOChunker(object):'
         result = None
         stop_iteration = None
         try:
-            result = self.output.next()
+            result = next(self.stdout)
         except StopIteration as e:
             stop_iteration = e
 
-        if self.process.poll() and self._fail_on_return_code:
-            err = '%s' % ''.join(self.error)
-            raise EnvironmentError(
-                "Subprocess exited due to an error:\n" + err)
+        if self.process:
+            return_code = self.process.poll()
+            ret_code_fail = return_code is not None and return_code != 0
+            if ret_code_fail and self._fail_on_return_code:
+                self.stop_streams()
+                err = self.get_stderr()
+                raise OSError(
+                    f"Subprocess exited (exit_code:{return_code}) due to an error during iteration:\n{err}")
 
         if stop_iteration:
             raise stop_iteration
         return result
 
-    def throw(self, type, value=None, traceback=None):
-        if self.output.length or not self.output.done_reading:
-            raise type(value)
+    def throw(self, exc_type, value=None, traceback=None):
+        if self.stdout.length or not self.stdout.done_reading:
+            raise exc_type(value)
 
     def close(self):
         if self._closed:
             return
-        self._closed = True
+
         try:
             self.process.terminate()
         except Exception:
@@ -477,11 +503,11 b' class SubprocessIOChunker(object):'
         if self._close_input_fd:
             os.close(self._close_input_fd)
         try:
-            self.output.close()
+            self.stdout.close()
         except Exception:
             pass
         try:
-            self.error.close()
+            self.stderr.close()
         except Exception:
             pass
         try:
@@ -489,6 +515,24 b' class SubprocessIOChunker(object):'
         except Exception:
             pass
 
+        self._closed = True
+
+    def stop_streams(self):
+        getattr(self.stdout, 'stop', lambda: None)()
+        getattr(self.stderr, 'stop', lambda: None)()
+
+    def get_stdout(self):
+        if self._stdout:
+            return self._stdout
+        else:
+            return b''.join(self.stdout)
+
+    def get_stderr(self):
+        if self._stderr:
+            return self._stderr
+        else:
+            return b''.join(self.stderr)
+
 
 def run_command(arguments, env=None):
     """
@@ -506,9 +550,9 b' def run_command(arguments, env=None):'
         if env:
             _opts.update({'env': env})
         proc = SubprocessIOChunker(cmd, **_opts)
-        return ''.join(proc), ''.join(proc.error)
-    except (EnvironmentError, OSError) as err:
-        cmd = ' '.join(cmd)  # human friendly CMD
+        return b''.join(proc), b''.join(proc.stderr)
+    except OSError as err:
+        cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
         tb_err = ("Couldn't run subprocessio command (%s).\n"
                   "Original error was:%s\n" % (cmd, err))
         log.exception(tb_err)

vcsserver/svn_diff.py

@@ -1,7 +1,7 b''
-# -*- coding: utf-8 -*-
 #
 # Copyright (C) 2004-2009 Edgewall Software
 # Copyright (C) 2004-2006 Christopher Lenz <[e-mail]>
+# Copyright (C) 2014-2023 RhodeCode GmbH
 # All rights reserved.
 #
 # This software is licensed as described in the file COPYING, which
@@ -17,15 +17,15 b''
 import difflib
 
 
-def get_filtered_hunks(fromlines, tolines, context=None,
-                       ignore_blank_lines=False, ignore_case=False,
-                       ignore_space_changes=False):
+def get_filtered_hunks(from_lines, to_lines, context=None,
+                       ignore_blank_lines: bool = False, ignore_case: bool = False,
+                       ignore_space_changes: bool = False):
     """Retrieve differences in the form of `difflib.SequenceMatcher`
     opcodes, grouped according to the ``context`` and ``ignore_*``
     parameters.
 
-    :param fromlines: list of lines corresponding to the old content
-    :param tolines: list of lines corresponding to the new content
+    :param from_lines: list of lines corresponding to the old content
+    :param to_lines: list of lines corresponding to the new content
     :param ignore_blank_lines: differences about empty lines only are ignored
     :param ignore_case: upper case / lower case only differences are ignored
     :param ignore_space_changes: differences in amount of spaces are ignored
@@ -37,27 +37,27 b' def get_filtered_hunks(fromlines, toline'
     to filter out the results will come straight from the
     SequenceMatcher.
     """
-    hunks = get_hunks(fromlines, tolines, context)
+    hunks = get_hunks(from_lines, to_lines, context)
     if ignore_space_changes or ignore_case or ignore_blank_lines:
-        hunks = filter_ignorable_lines(hunks, fromlines, tolines, context,
+        hunks = filter_ignorable_lines(hunks, from_lines, to_lines, context,
                                        ignore_blank_lines, ignore_case,
                                        ignore_space_changes)
     return hunks
 
 
-def get_hunks(fromlines, tolines, context=None):
+def get_hunks(from_lines, to_lines, context=None):
     """Generator yielding grouped opcodes describing differences .
 
     See `get_filtered_hunks` for the parameter descriptions.
     """
-    matcher = difflib.SequenceMatcher(None, fromlines, tolines)
+    matcher = difflib.SequenceMatcher(None, from_lines, to_lines)
     if context is None:
         return (hunk for hunk in [matcher.get_opcodes()])
     else:
         return matcher.get_grouped_opcodes(context)
 
 
-def filter_ignorable_lines(hunks, fromlines, tolines, context,
+def filter_ignorable_lines(hunks, from_lines, to_lines, context,
                            ignore_blank_lines, ignore_case,
                            ignore_space_changes):
     """Detect line changes that should be ignored and emits them as
@@ -67,11 +67,12 b' def filter_ignorable_lines(hunks, fromli'
     See `get_filtered_hunks` for the parameter descriptions.
     """
     def is_ignorable(tag, fromlines, tolines):
+
         if tag == 'delete' and ignore_blank_lines:
-            if ''.join(fromlines) == '':
+            if b''.join(fromlines) == b'':
                 return True
         elif tag == 'insert' and ignore_blank_lines:
-            if ''.join(tolines) == '':
+            if b''.join(tolines) == b'':
                 return True
         elif tag == 'replace' and (ignore_case or ignore_space_changes):
             if len(fromlines) != len(tolines):
@@ -81,7 +82,7 b' def filter_ignorable_lines(hunks, fromli'
                 if ignore_case:
                     input_str = input_str.lower()
                 if ignore_space_changes:
-                    input_str = ' '.join(input_str.split())
+                    input_str = b' '.join(input_str.split())
                 return input_str
 
             for i in range(len(fromlines)):
@@ -101,7 +102,7 b' def filter_ignorable_lines(hunks, fromli'
                 else:
                     prev = (tag, i1, i2, j1, j2)
             else:
-                if is_ignorable(tag, fromlines[i1:i2], tolines[j1:j2]):
+                if is_ignorable(tag, from_lines[i1:i2], to_lines[j1:j2]):
                     ignored_lines = True
                     if prev:
                         prev = 'equal', prev[1], i2, prev[3], j2
@@ -125,10 +126,11 b' def filter_ignorable_lines(hunks, fromli'
             nn = n + n
 
             group = []
+
             def all_equal():
                 all(op[0] == 'equal' for op in group)
             for idx, (tag, i1, i2, j1, j2) in enumerate(opcodes):
-                if idx == 0 and tag == 'equal': # Fixup leading unchanged block
+                if idx == 0 and tag == 'equal':  # Fixup leading unchanged block
                     i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
                 elif tag == 'equal' and i2 - i1 > nn:
                     group.append((tag, i1, min(i2, i1 + n), j1,
@@ -140,7 +142,7 b' def filter_ignorable_lines(hunks, fromli'
                 group.append((tag, i1, i2, j1, j2))
 
             if group and not (len(group) == 1 and group[0][0] == 'equal'):
-                if group[-1][0] == 'equal': # Fixup trailing unchanged block
+                if group[-1][0] == 'equal':  # Fixup trailing unchanged block
                     tag, i1, i2, j1, j2 = group[-1]
                     group[-1] = tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)
                 if not all_equal():
@@ -150,22 +152,30 b' def filter_ignorable_lines(hunks, fromli'
             yield hunk
 
 
-NO_NEWLINE_AT_END = '\\ No newline at end of file'
+NO_NEWLINE_AT_END = b'\\ No newline at end of file'
+LINE_TERM = b'\n'
 
 
-def unified_diff(fromlines, tolines, context=None, ignore_blank_lines=0,
-                 ignore_case=0, ignore_space_changes=0, lineterm='\n'):
+def unified_diff(from_lines, to_lines, context=None, ignore_blank_lines: bool = False,
+                 ignore_case: bool = False, ignore_space_changes: bool = False, lineterm=LINE_TERM) -> bytes:
     """
     Generator producing lines corresponding to a textual diff.
 
     See `get_filtered_hunks` for the parameter descriptions.
     """
     # TODO: johbo: Check if this can be nicely integrated into the matching
+
     if ignore_space_changes:
-        fromlines = [l.strip() for l in fromlines]
-        tolines = [l.strip() for l in tolines]
+        from_lines = [l.strip() for l in from_lines]
+        to_lines = [l.strip() for l in to_lines]
 
-    for group in get_filtered_hunks(fromlines, tolines, context,
+    def _hunk_range(start, length) -> bytes:
+        if length != 1:
+            return b'%d,%d' % (start, length)
+        else:
+            return b'%d' % (start,)
+
+    for group in get_filtered_hunks(from_lines, to_lines, context,
                                     ignore_blank_lines, ignore_case,
                                     ignore_space_changes):
         i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
@@ -173,37 +183,30 b' def unified_diff(fromlines, tolines, con'
             i1, i2 = -1, -1  # support for Add changes
         if j1 == 0 and j2 == 0:
             j1, j2 = -1, -1  # support for Delete changes
-        yield '@@ -%s +%s @@%s' % (
+        yield b'@@ -%b +%b @@%b' % (
             _hunk_range(i1 + 1, i2 - i1),
             _hunk_range(j1 + 1, j2 - j1),
             lineterm)
         for tag, i1, i2, j1, j2 in group:
             if tag == 'equal':
-                for line in fromlines[i1:i2]:
+                for line in from_lines[i1:i2]:
                     if not line.endswith(lineterm):
-                        yield ' ' + line + lineterm
+                        yield b' ' + line + lineterm
                         yield NO_NEWLINE_AT_END + lineterm
                     else:
-                        yield ' ' + line
+                        yield b' ' + line
             else:
                 if tag in ('replace', 'delete'):
-                    for line in fromlines[i1:i2]:
+                    for line in from_lines[i1:i2]:
                         if not line.endswith(lineterm):
-                            yield '-' + line + lineterm
+                            yield b'-' + line + lineterm
                             yield NO_NEWLINE_AT_END + lineterm
                         else:
-                            yield '-' + line
+                            yield b'-' + line
                 if tag in ('replace', 'insert'):
-                    for line in tolines[j1:j2]:
+                    for line in to_lines[j1:j2]:
                         if not line.endswith(lineterm):
-                            yield '+' + line + lineterm
+                            yield b'+' + line + lineterm
                             yield NO_NEWLINE_AT_END + lineterm
                         else:
-                            yield '+' + line
-
-
-def _hunk_range(start, length):
-    if length != 1:
-        return '%d,%d' % (start, length)
-    else:
-        return '%d' % (start, )
+                            yield b'+' + line

vcsserver/tests/__init__.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/tests/fixture.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,8 +18,7 b''
 import os
 import shutil
 import tempfile
-
-import configobj
+import configparser
 
 
 class ContextINI(object):
@@ -53,17 +52,17 b' class ContextINI(object):'
             with open(self.new_path, 'wb'):
                 pass
 
-        config = configobj.ConfigObj(
-            self.new_path, file_error=True, write_empty_values=True)
+        parser = configparser.ConfigParser()
+        parser.read(self.ini_file_path)
 
         for data in self.ini_params:
-            section, ini_params = data.items()[0]
-            key, val = ini_params.items()[0]
-            if section not in config:
-                config[section] = {}
-            config[section][key] = val
-
-        config.write()
+            section, ini_params = list(data.items())[0]
+            key, val = list(ini_params.items())[0]
+            if section not in parser:
+                parser[section] = {}
+            parser[section][key] = val
+        with open(self.ini_file_path, 'w') as f:
+            parser.write(f)
         return self.new_path
 
     def __exit__(self, exc_type, exc_val, exc_tb):

vcsserver/tests/test_git.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,8 +21,7 b' import pytest'
 import dulwich.errors
 from mock import Mock, patch
 
-from vcsserver import git
-
+from vcsserver.remote import git
 
 SAMPLE_REFS = {
     'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
@@ -49,7 +48,7 b' def test_discover_git_version(git_remote'
 
 
 class TestGitFetch(object):
-    def setup(self):
+    def setup_method(self):
         self.mock_repo = Mock()
         factory = Mock()
         factory.repo = Mock(return_value=self.mock_repo)
@@ -67,8 +66,8 b' class TestGitFetch(object):'
 
     def test_fetches_specified_commits(self):
         selected_refs = {
-            'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
-            'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
+            'refs/tags/v0.1.8': b'74ebce002c088b8a5ecf40073db09375515ecd68',
+            'refs/tags/v0.1.3': b'5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         def side_effect(determine_wants, *args, **kwargs):
@@ -80,7 +79,7 b' class TestGitFetch(object):'
             mock_fetch.side_effect = side_effect
             self.remote_git.pull(
                 wire={}, url='/tmp/', apply_refs=False,
-                refs=selected_refs.keys())
+                refs=list(selected_refs.keys()))
             determine_wants = self.mock_repo.object_store.determine_wants_all
             assert determine_wants.call_count == 0
 
@@ -93,7 +92,7 b' class TestGitFetch(object):'
             'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
-        with patch('vcsserver.git.Repo', create=False) as mock_repo:
+        with patch('vcsserver.remote.git.Repo', create=False) as mock_repo:
             mock_repo().get_refs.return_value = sample_refs
             remote_refs = remote_git.get_remote_refs(wire={}, url=url)
             mock_repo().get_refs.assert_called_once_with()
@@ -114,7 +113,7 b' class TestReraiseSafeExceptions(object):'
         methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
         for method_name, method in methods:
             if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
-                assert method.im_func.__code__ == decorator.__code__
+                assert method.__func__.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
         (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
@@ -138,11 +137,14 b' class TestReraiseSafeExceptions(object):'
 class TestDulwichRepoWrapper(object):
     def test_calls_close_on_delete(self):
         isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
-        with isdir_patcher:
-            repo = git.Repo('/tmp/abcde')
-        with patch.object(git.DulwichRepo, 'close') as close_mock:
-            del repo
-        close_mock.assert_called_once_with()
+        with patch.object(git.Repo, 'close') as close_mock:
+            with isdir_patcher:
+                repo = git.Repo('/tmp/abcde')
+                assert repo is not None
+                repo.__del__()
+                # can't use del repo as in python3 this isn't always calling .__del__()
+
+            close_mock.assert_called_once_with()
 
 
 class TestGitFactory(object):

vcsserver/tests/test_hg.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,9 +21,10 b' import traceback'
 
 import pytest
 from mercurial.error import LookupError
-from mock import Mock, MagicMock, patch
+from mock import Mock, patch
 
-from vcsserver import exceptions, hg, hgcompat
+from vcsserver import exceptions, hgcompat
+from vcsserver.remote import hg
 
 
 class TestDiff(object):
@@ -32,8 +33,8 b' class TestDiff(object):'
         factory = Mock()
         hg_remote = hg.HgRemote(factory)
         with patch('mercurial.patch.diff') as diff_mock:
-            diff_mock.side_effect = LookupError(
-                'deadbeef', 'index', 'message')
+            diff_mock.side_effect = LookupError(b'deadbeef', b'index', b'message')
+
             with pytest.raises(Exception) as exc_info:
                 hg_remote.diff(
                     wire={}, commit_id_1='deadbeef', commit_id_2='deadbee1',
@@ -51,13 +52,13 b' class TestReraiseSafeExceptions(object):'
         decorator = hg.reraise_safe_exceptions(None)
         for method_name, method in methods:
             if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']:
-                assert method.im_func.__code__ == decorator.__code__
+                assert method.__func__.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
-        (hgcompat.Abort(), 'abort'),
-        (hgcompat.InterventionRequired(), 'abort'),
+        (hgcompat.Abort('failed-abort'), 'abort'),
+        (hgcompat.InterventionRequired('intervention-required'), 'abort'),
         (hgcompat.RepoLookupError(), 'lookup'),
-        (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
+        (hgcompat.LookupError(b'deadbeef', b'index', b'message'), 'lookup'),
         (hgcompat.RepoError(), 'error'),
         (hgcompat.RequirementError(), 'requirement'),
     ])
@@ -75,10 +76,9 b' class TestReraiseSafeExceptions(object):'
         @hg.reraise_safe_exceptions
         def fake_method():
             try:
-                raise hgcompat.Abort()
+                raise hgcompat.Abort('test-abort')
             except:
-                self.original_traceback = traceback.format_tb(
-                    sys.exc_info()[2])
+                self.original_traceback = traceback.format_tb(sys.exc_info()[2])
                 raise
 
         try:

vcsserver/tests/test_hgpatches.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,14 +21,14 b' import pytest'
 from vcsserver import hgcompat, hgpatches
 
 
-LARGEFILES_CAPABILITY = 'largefiles=serve'
+LARGEFILES_CAPABILITY = b'largefiles=serve'
 
 
 def test_patch_largefiles_capabilities_applies_patch(
         patched_capabilities):
     lfproto = hgcompat.largefiles.proto
     hgpatches.patch_largefiles_capabilities()
-    assert lfproto._capabilities.func_name == '_dynamic_capabilities'
+    assert lfproto._capabilities.__name__ == '_dynamic_capabilities'
 
 
 def test_dynamic_capabilities_uses_original_function_if_not_enabled(
@@ -72,11 +72,6 b' def test_dynamic_capabilities_uses_large'
     assert LARGEFILES_CAPABILITY in caps
 
 
-def test_hgsubversion_import():
-    from hgsubversion import svnrepo
-    assert svnrepo
-
-
 @pytest.fixture
 def patched_capabilities(request):
     """

vcsserver/tests/test_hooks.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,17 +15,18 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
-import contextlib
-import io
 import threading
-from BaseHTTPServer import BaseHTTPRequestHandler
-from SocketServer import TCPServer
+import msgpack
+
+from http.server import BaseHTTPRequestHandler
+from socketserver import TCPServer
 
 import mercurial.ui
 import mock
 import pytest
-import simplejson as json
 
+from vcsserver.hooks import HooksHttpClient
+from vcsserver.lib.rc_json import json
 from vcsserver import hooks
 
 
@@ -44,7 +45,7 b' def get_hg_ui(extras=None):'
     }
     required_extras.update(extras)
     hg_ui = mercurial.ui.ui()
-    hg_ui.setconfig('rhodecode', 'RC_SCM_DATA', json.dumps(required_extras))
+    hg_ui.setconfig(b'rhodecode', b'RC_SCM_DATA', json.dumps(required_extras))
 
     return hg_ui
 
@@ -67,6 +68,7 b' def test_git_post_receive_is_disabled():'
 
 def test_git_post_receive_calls_repo_size():
     extras = {'hooks': ['push', 'repo_size']}
+
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
@@ -81,6 +83,7 b' def test_git_post_receive_calls_repo_siz'
 
 def test_git_post_receive_does_not_call_disabled_repo_size():
     extras = {'hooks': ['push']}
+
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
@@ -149,18 +152,19 b' class TestHooksHttpClient(object):'
         client = hooks.HooksHttpClient(uri)
         assert client.hooks_uri == uri
 
-    def test_serialize_returns_json_string(self):
+    def test_serialize_returns_serialized_string(self):
         client = hooks.HooksHttpClient('localhost:3000')
         hook_name = 'test'
         extras = {
             'first': 1,
             'second': 'two'
         }
-        result = client._serialize(hook_name, extras)
-        expected_result = json.dumps({
+        hooks_proto, result = client._serialize(hook_name, extras)
+        expected_result = msgpack.packb({
             'method': hook_name,
-            'extras': extras
+            'extras': extras,
         })
+        assert hooks_proto == {'rc-hooks-protocol': 'msgpack.v1', 'Connection': 'keep-alive'}
         assert result == expected_result
 
     def test_call_queries_http_server(self, http_mirror):
@@ -171,10 +175,10 b' class TestHooksHttpClient(object):'
             'second': 'two'
         }
         result = client(hook_name, extras)
-        expected_result = {
+        expected_result = msgpack.unpackb(msgpack.packb({
             'method': hook_name,
             'extras': extras
-        }
+        }), raw=False)
         assert result == expected_result
 
 
@@ -211,9 +215,10 b' def http_mirror(request):'
 
 
 class MirrorHttpHandler(BaseHTTPRequestHandler):
+
     def do_POST(self):
         length = int(self.headers['Content-Length'])
-        body = self.rfile.read(length).decode('utf-8')
+        body = self.rfile.read(length)
         self.send_response(200)
         self.end_headers()
         self.wfile.write(body)
@@ -239,3 +244,43 b' class MirrorHttpServer(object):'
     @property
     def uri(self):
         return '{}:{}'.format(self.ip_address, self.port)
+
+
+def test_hooks_http_client_init():
+    hooks_uri = 'http://localhost:8000'
+    client = HooksHttpClient(hooks_uri)
+    assert client.hooks_uri == hooks_uri
+
+
+def test_hooks_http_client_call():
+    hooks_uri = 'http://localhost:8000'
+
+    method = 'test_method'
+    extras = {'key': 'value'}
+
+    with \
+        mock.patch('http.client.HTTPConnection') as mock_connection,\
+        mock.patch('msgpack.load') as mock_load:
+
+        client = HooksHttpClient(hooks_uri)
+
+        mock_load.return_value = {'result': 'success'}
+        response = mock.MagicMock()
+        response.status = 200
+        mock_connection.request.side_effect = None
+        mock_connection.getresponse.return_value = response
+
+        result = client(method, extras)
+
+        mock_connection.assert_called_with(hooks_uri)
+        mock_connection.return_value.request.assert_called_once()
+        assert result == {'result': 'success'}
+
+
+def test_hooks_http_client_serialize():
+    method = 'test_method'
+    extras = {'key': 'value'}
+    headers, body = HooksHttpClient._serialize(method, extras)
+
+    assert headers == {'rc-hooks-protocol': HooksHttpClient.proto, 'Connection': 'keep-alive'}
+    assert msgpack.unpackb(body) == {'method': method, 'extras': extras}

vcsserver/tests/test_http_performance.py

@@ -30,13 +30,13 b' def data():'
 
 def test_http_app_streaming_with_data(data, repeat, vcs_app):
     app = vcs_app
-    for x in xrange(repeat / 10):
+    for x in range(repeat // 10):
         response = app.post('/stream/git/', params=data)
         assert response.status_code == 200
 
 
 def test_http_app_streaming_no_data(repeat, vcs_app):
     app = vcs_app
-    for x in xrange(repeat / 10):
+    for x in range(repeat // 10):
         response = app.post('/stream/git/')
         assert response.status_code == 200

vcsserver/tests/test_install_hooks.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 b' import vcsserver'
 import tempfile
 from vcsserver import hook_utils
 from vcsserver.tests.fixture import no_newline_id_generator
+from vcsserver.str_utils import safe_bytes, safe_str
 from vcsserver.utils import AttributeDict
 
 
@@ -31,7 +32,7 b' class TestCheckRhodecodeHook(object):'
     def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir):
         hook = os.path.join(str(tmpdir), 'fake_hook_file.py')
         with open(hook, 'wb') as f:
-            f.write('dummy test')
+            f.write(b'dummy test')
             result = hook_utils.check_rhodecode_hook(hook)
             assert result is False
 
@@ -47,7 +48,7 b' class TestCheckRhodecodeHook(object):'
     def test_signatures(self, file_content, expected_result, tmpdir):
         hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py')
         with open(hook, 'wb') as f:
-            f.write(file_content)
+            f.write(safe_bytes(file_content))
 
         result = hook_utils.check_rhodecode_hook(hook)
 
@@ -71,8 +72,7 b' class BaseInstallHooks(object):'
             content = hook_file.read()
 
         expected_env = '#!{}'.format(executable)
-        expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(
-            vcsserver.__version__)
+        expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(vcsserver.__version__)
         assert content.strip().startswith(expected_env)
         assert expected_rc_version in content
 

vcsserver/tests/test_main_http.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -42,8 +42,7 b' def test_applies_largefiles_patch_only_i'
     ('bad', 'bad'),
     ('query&foo=bar', 'query&foo=bar'),
     ('equery&auth_token=bar', 'equery&auth_token=*****'),
-    ('a;b;c;query&foo=bar&auth_token=secret',
-     'a&b&c&query&foo=bar&auth_token=*****'),
+    ('a;b;c;query&foo=bar&auth_token=secret', 'a;b;c;query&foo=bar&auth_token=*****'),
     ('', ''),
     (None, None),
     ('foo=bar', 'foo=bar'),

vcsserver/tests/test_pygrack.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,6 +16,7 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
+import more_itertools
 
 import dulwich.protocol
 import mock
@@ -25,7 +26,7 b' import webtest'
 
 from vcsserver import hooks, pygrack
 
-# pylint: disable=redefined-outer-name,protected-access
+from vcsserver.str_utils import ascii_bytes
 
 
 @pytest.fixture()
@@ -75,8 +76,7 b' def test_pre_pull_hook_fails_with_sideba'
         '0000',
         '0009done\n',
     ])
-    with mock.patch('vcsserver.hooks.git_pre_pull',
-                    return_value=hooks.HookResponse(1, 'foo')):
+    with mock.patch('vcsserver.hooks.git_pre_pull', return_value=hooks.HookResponse(1, 'foo')):
         response = pygrack_app.post(
             '/git-upload-pack', params=request,
             content_type='application/x-git-upload-pack')
@@ -86,8 +86,8 b' def test_pre_pull_hook_fails_with_sideba'
     packets = list(proto.read_pkt_seq())
 
     expected_packets = [
-        'NAK\n', '\x02foo', '\x02Pre pull hook failed: aborting\n',
-        '\x01' + pygrack.GitRepository.EMPTY_PACK,
+        b'NAK\n', b'\x02foo', b'\x02Pre pull hook failed: aborting\n',
+        b'\x01' + pygrack.GitRepository.EMPTY_PACK,
     ]
     assert packets == expected_packets
 
@@ -115,12 +115,16 b' def test_pull_has_hook_messages(pygrack_'
         '0000',
         '0009done\n',
     ])
+
+    pre_pull = 'pre_pull_output'
+    post_pull = 'post_pull_output'
+
     with mock.patch('vcsserver.hooks.git_pre_pull',
-                    return_value=hooks.HookResponse(0, 'foo')):
+                    return_value=hooks.HookResponse(0, pre_pull)):
         with mock.patch('vcsserver.hooks.git_post_pull',
-                        return_value=hooks.HookResponse(1, 'bar')):
+                        return_value=hooks.HookResponse(1, post_pull)):
             with mock.patch('vcsserver.subprocessio.SubprocessIOChunker',
-                            return_value=['0008NAK\n0009subp\n0000']):
+                            return_value=more_itertools.always_iterable([b'0008NAK\n0009subp\n0000'])):
                 response = pygrack_app.post(
                     '/git-upload-pack', params=request,
                     content_type='application/x-git-upload-pack')
@@ -129,13 +133,17 b' def test_pull_has_hook_messages(pygrack_'
     proto = dulwich.protocol.Protocol(data.read, None)
     packets = list(proto.read_pkt_seq())
 
-    assert packets == ['NAK\n', '\x02foo', 'subp\n', '\x02bar']
+    assert packets == [b'NAK\n',
+                       # pre-pull only outputs if IT FAILS as in != 0 ret code
+                       #b'\x02pre_pull_output',
+                       b'subp\n',
+                       b'\x02post_pull_output']
 
 
 def test_get_want_capabilities(pygrack_instance):
     data = io.BytesIO(
-        '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
-        'multi_ack side-band-64k ofs-delta\n00000009done\n')
+        b'0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
+        b'multi_ack side-band-64k ofs-delta\n00000009done\n')
 
     request = webob.Request({
         'wsgi.input': data,
@@ -146,20 +154,20 b' def test_get_want_capabilities(pygrack_i'
     capabilities = pygrack_instance._get_want_capabilities(request)
 
     assert capabilities == frozenset(
-        ('ofs-delta', 'multi_ack', 'side-band-64k'))
+        (b'ofs-delta', b'multi_ack', b'side-band-64k'))
     assert data.tell() == 0
 
 
 @pytest.mark.parametrize('data,capabilities,expected', [
     ('foo', [], []),
-    ('', ['side-band-64k'], []),
-    ('', ['side-band'], []),
-    ('foo', ['side-band-64k'], ['0008\x02foo']),
-    ('foo', ['side-band'], ['0008\x02foo']),
-    ('f'*1000, ['side-band-64k'], ['03ed\x02' + 'f' * 1000]),
-    ('f'*1000, ['side-band'], ['03e8\x02' + 'f' * 995, '000a\x02fffff']),
-    ('f'*65520, ['side-band-64k'], ['fff0\x02' + 'f' * 65515, '000a\x02fffff']),
-    ('f'*65520, ['side-band'], ['03e8\x02' + 'f' * 995] * 65 + ['0352\x02' + 'f' * 845]),
+    ('', [pygrack.CAPABILITY_SIDE_BAND_64K], []),
+    ('', [pygrack.CAPABILITY_SIDE_BAND], []),
+    ('foo', [pygrack.CAPABILITY_SIDE_BAND_64K], [b'0008\x02foo']),
+    ('foo', [pygrack.CAPABILITY_SIDE_BAND], [b'0008\x02foo']),
+    ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'03ed\x02' + b'f' * 1000]),
+    ('f'*1000, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995, b'000a\x02fffff']),
+    ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND_64K], [b'fff0\x02' + b'f' * 65515, b'000a\x02fffff']),
+    ('f'*65520, [pygrack.CAPABILITY_SIDE_BAND], [b'03e8\x02' + b'f' * 995] * 65 + [b'0352\x02' + b'f' * 845]),
 ], ids=[
     'foo-empty',
     'empty-64k', 'empty',
@@ -174,54 +182,59 b' def test_get_messages(pygrack_instance, '
 
 @pytest.mark.parametrize('response,capabilities,pre_pull_messages,post_pull_messages', [
     # Unexpected response
-    ('unexpected_response', ['side-band-64k'], 'foo', 'bar'),
+    ([b'unexpected_response[no_initial_header]'], [pygrack.CAPABILITY_SIDE_BAND_64K], 'foo', 'bar'),
     # No sideband
-    ('no-sideband', [], 'foo', 'bar'),
+    ([b'no-sideband'], [], 'foo', 'bar'),
     # No messages
-    ('no-messages', ['side-band-64k'], '', ''),
+    ([b'no-messages'], [pygrack.CAPABILITY_SIDE_BAND_64K], '', ''),
 ])
 def test_inject_messages_to_response_nothing_to_do(
-        pygrack_instance, response, capabilities, pre_pull_messages,
-        post_pull_messages):
-    new_response = pygrack_instance._inject_messages_to_response(
-        response, capabilities, pre_pull_messages, post_pull_messages)
+        pygrack_instance, response, capabilities, pre_pull_messages, post_pull_messages):
 
-    assert new_response == response
+    new_response = pygrack_instance._build_post_pull_response(
+        more_itertools.always_iterable(response), capabilities, pre_pull_messages, post_pull_messages)
+
+    assert list(new_response) == response
 
 
 @pytest.mark.parametrize('capabilities', [
-    ['side-band'],
-    ['side-band-64k'],
+    [pygrack.CAPABILITY_SIDE_BAND],
+    [pygrack.CAPABILITY_SIDE_BAND_64K],
 ])
-def test_inject_messages_to_response_single_element(pygrack_instance,
-                                                    capabilities):
-    response = ['0008NAK\n0009subp\n0000']
-    new_response = pygrack_instance._inject_messages_to_response(
-        response, capabilities, 'foo', 'bar')
+def test_inject_messages_to_response_single_element(pygrack_instance, capabilities):
+    response = [b'0008NAK\n0009subp\n0000']
+    new_response = pygrack_instance._build_post_pull_response(
+        more_itertools.always_iterable(response), capabilities, 'foo', 'bar')
 
-    expected_response = [
-        '0008NAK\n', '0008\x02foo', '0009subp\n', '0008\x02bar', '0000']
+    expected_response = b''.join([
+        b'0008NAK\n',
+        b'0008\x02foo',
+        b'0009subp\n',
+        b'0008\x02bar',
+        b'0000'])
 
-    assert new_response == expected_response
+    assert b''.join(new_response) == expected_response
 
 
 @pytest.mark.parametrize('capabilities', [
-    ['side-band'],
-    ['side-band-64k'],
+    [pygrack.CAPABILITY_SIDE_BAND],
+    [pygrack.CAPABILITY_SIDE_BAND_64K],
 ])
-def test_inject_messages_to_response_multi_element(pygrack_instance,
-                                                   capabilities):
-    response = [
-        '0008NAK\n000asubp1\n', '000asubp2\n', '000asubp3\n', '000asubp4\n0000']
-    new_response = pygrack_instance._inject_messages_to_response(
-        response, capabilities, 'foo', 'bar')
+def test_inject_messages_to_response_multi_element(pygrack_instance, capabilities):
+    response = more_itertools.always_iterable([
+        b'0008NAK\n000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n0000'
+    ])
+    new_response = pygrack_instance._build_post_pull_response(response, capabilities, 'foo', 'bar')
 
-    expected_response = [
-        '0008NAK\n', '0008\x02foo', '000asubp1\n', '000asubp2\n', '000asubp3\n',
-        '000asubp4\n', '0008\x02bar', '0000'
-    ]
+    expected_response = b''.join([
+        b'0008NAK\n',
+        b'0008\x02foo',
+        b'000asubp1\n', b'000asubp2\n', b'000asubp3\n', b'000asubp4\n',
+        b'0008\x02bar',
+        b'0000'
+    ])
 
-    assert new_response == expected_response
+    assert b''.join(new_response) == expected_response
 
 
 def test_build_failed_pre_pull_response_no_sideband(pygrack_instance):
@@ -231,19 +244,52 b' def test_build_failed_pre_pull_response_'
 
 
 @pytest.mark.parametrize('capabilities', [
-    ['side-band'],
-    ['side-band-64k'],
-    ['side-band-64k', 'no-progress'],
+    [pygrack.CAPABILITY_SIDE_BAND],
+    [pygrack.CAPABILITY_SIDE_BAND_64K],
+    [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'],
 ])
 def test_build_failed_pre_pull_response(pygrack_instance, capabilities):
-    response = pygrack_instance._build_failed_pre_pull_response(
-        capabilities, 'foo')
+    response = pygrack_instance._build_failed_pre_pull_response(capabilities, 'foo')
 
     expected_response = [
-        '0008NAK\n', '0008\x02foo', '0024\x02Pre pull hook failed: aborting\n',
-        '%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5,
-                        pygrack.GitRepository.EMPTY_PACK),
-        '0000',
+        b'0008NAK\n', b'0008\x02foo', b'0024\x02Pre pull hook failed: aborting\n',
+        b'%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5, pygrack.GitRepository.EMPTY_PACK),
+        pygrack.GitRepository.FLUSH_PACKET,
     ]
 
     assert response == expected_response
+
+
+def test_inject_messages_to_response_generator(pygrack_instance):
+
+    def response_generator():
+        response = [
+            # protocol start
+            b'0008NAK\n',
+        ]
+        response += [ascii_bytes(f'000asubp{x}\n') for x in range(1000)]
+        response += [
+            # protocol end
+            pygrack.GitRepository.FLUSH_PACKET
+        ]
+        for elem in response:
+            yield elem
+
+    new_response = pygrack_instance._build_post_pull_response(
+        response_generator(), [pygrack.CAPABILITY_SIDE_BAND_64K, b'no-progress'], 'PRE_PULL_MSG\n', 'POST_PULL_MSG\n')
+
+    assert iter(new_response)
+
+    expected_response = b''.join([
+        # start
+        b'0008NAK\n0012\x02PRE_PULL_MSG\n',
+    ] + [
+        # ... rest
+        ascii_bytes(f'000asubp{x}\n') for x in range(1000)
+    ] + [
+        # final message,
+        b'0013\x02POST_PULL_MSG\n0000',
+
+    ])
+
+    assert b''.join(new_response) == expected_response

vcsserver/tests/test_scm_app.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,10 +25,11 b' import pytest'
 import webtest
 
 from vcsserver import scm_app
+from vcsserver.str_utils import ascii_bytes
 
 
 def test_hg_does_not_accept_invalid_cmd(tmpdir):
-    repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
+    repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
     app = webtest.TestApp(scm_app.HgWeb(repo))
 
     response = app.get('/repo?cmd=invalidcmd', expect_errors=True)
@@ -37,7 +38,7 b' def test_hg_does_not_accept_invalid_cmd('
 
 
 def test_create_hg_wsgi_app_requirement_error(tmpdir):
-    repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
+    repo = mercurial.hg.repository(mercurial.ui.ui(), ascii_bytes(str(tmpdir)), create=True)
     config = (
         ('paths', 'default', ''),
     )

vcsserver/tests/test_server.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

vcsserver/tests/test_subprocessio.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -22,22 +22,23 b' import sys'
 import pytest
 
 from vcsserver import subprocessio
+from vcsserver.str_utils import ascii_bytes
 
 
-class KindaFilelike(object):  # pragma: no cover
+class FileLikeObj(object):  # pragma: no cover
 
-    def __init__(self, data, size):
-        chunks = size / len(data)
+    def __init__(self, data: bytes, size):
+        chunks = size // len(data)
 
         self.stream = self._get_stream(data, chunks)
 
     def _get_stream(self, data, chunks):
-        for x in xrange(chunks):
+        for x in range(chunks):
             yield data
 
     def read(self, n):
 
-        buffer_stream = ''
+        buffer_stream = b''
         for chunk in self.stream:
             buffer_stream += chunk
             if len(buffer_stream) >= n:
@@ -51,7 +52,7 b' class KindaFilelike(object): # pragma: '
 def environ():
     """Delete coverage variables, as they make the tests fail."""
     env = dict(os.environ)
-    for key in env.keys():
+    for key in list(env.keys()):
         if key.startswith('COV_CORE_'):
             del env[key]
 
@@ -63,93 +64,92 b' def _get_python_args(script):'
 
 
 def test_raise_exception_on_non_zero_return_code(environ):
-    args = _get_python_args('sys.exit(1)')
-    with pytest.raises(EnvironmentError):
-        list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
+    call_args = _get_python_args('raise ValueError("fail")')
+    with pytest.raises(OSError):
+        b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
 
 
 def test_does_not_fail_on_non_zero_return_code(environ):
-    args = _get_python_args('sys.exit(1)')
-    output = ''.join(
-        subprocessio.SubprocessIOChunker(
-            args, shell=False, fail_on_return_code=False, env=environ
-        )
-    )
+    call_args = _get_python_args('sys.stdout.write("hello"); sys.exit(1)')
+    proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
+    output = b''.join(proc)
 
-    assert output == ''
+    assert output == b'hello'
 
 
 def test_raise_exception_on_stderr(environ):
-    args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
-    with pytest.raises(EnvironmentError) as excinfo:
-        list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
+    call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); time.sleep(1);')
 
-    assert 'exited due to an error:\nX' in str(excinfo.value)
+    with pytest.raises(OSError) as excinfo:
+        b''.join(subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ))
+
+    assert 'exited due to an error:\nWRITE_TO_STDERR' in str(excinfo.value)
 
 
 def test_does_not_fail_on_stderr(environ):
-    args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
-    output = ''.join(
-        subprocessio.SubprocessIOChunker(
-            args, shell=False, fail_on_stderr=False, env=environ
-        )
-    )
+    call_args = _get_python_args('sys.stderr.write("WRITE_TO_STDERR"); sys.stderr.flush; time.sleep(2);')
+    proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_stderr=False, env=environ)
+    output = b''.join(proc)
 
-    assert output == ''
+    assert output == b''
 
 
[e-mail]('size', [1, 10 ** 5])
[e-mail]('size', [
+    1,
+    10 ** 5
+])
 def test_output_with_no_input(size, environ):
-    print(type(environ))
-    data = 'X'
-    args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size))
-    output = ''.join(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
+    call_args = _get_python_args(f'sys.stdout.write("X" * {size});')
+    proc = subprocessio.SubprocessIOChunker(call_args, shell=False, env=environ)
+    output = b''.join(proc)
 
-    assert output == data * size
+    assert output == ascii_bytes("X" * size)
 
 
[e-mail]('size', [1, 10 ** 5])
[e-mail]('size', [
+    1,
+    10 ** 5
+])
 def test_output_with_no_input_does_not_fail(size, environ):
-    data = 'X'
-    args = _get_python_args('sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
-    output = ''.join(
-        subprocessio.SubprocessIOChunker(
-            args, shell=False, fail_on_return_code=False, env=environ
-        )
-    )
 
-    print("{} {}".format(len(data * size), len(output)))
-    assert output == data * size
+    call_args = _get_python_args(f'sys.stdout.write("X" * {size}); sys.exit(1)')
+    proc = subprocessio.SubprocessIOChunker(call_args, shell=False, fail_on_return_code=False, env=environ)
+    output = b''.join(proc)
+
+    assert output == ascii_bytes("X" * size)
 
 
[e-mail]('size', [1, 10 ** 5])
[e-mail]('size', [
+    1,
+    10 ** 5
+])
 def test_output_with_input(size, environ):
     data_len = size
-    inputstream = KindaFilelike('X', size)
+    inputstream = FileLikeObj(b'X', size)
 
     # This acts like the cat command.
-    args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
-    output = ''.join(
-        subprocessio.SubprocessIOChunker(
-            args, shell=False, inputstream=inputstream, env=environ
-        )
+    call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
+    # note: in this tests we explicitly don't assign chunker to a variable and let it stream directly
+    output = b''.join(
+        subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
     )
 
     assert len(output) == data_len
 
 
[e-mail]('size', [1, 10 ** 5])
[e-mail]('size', [
+    1,
+    10 ** 5
+])
 def test_output_with_input_skipping_iterator(size, environ):
     data_len = size
-    inputstream = KindaFilelike('X', size)
+    inputstream = FileLikeObj(b'X', size)
 
     # This acts like the cat command.
-    args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
+    call_args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
 
     # Note: assigning the chunker makes sure that it is not deleted too early
-    chunker = subprocessio.SubprocessIOChunker(
-        args, shell=False, inputstream=inputstream, env=environ
-    )
-    output = ''.join(chunker.output)
+    proc = subprocessio.SubprocessIOChunker(call_args, shell=False, input_stream=inputstream, env=environ)
+    output = b''.join(proc.stdout)
 
     assert len(output) == data_len

vcsserver/tests/test_svn.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,10 +20,12 b' import mock'
 import pytest
 import sys
 
+from vcsserver.str_utils import ascii_bytes
+
 
 class MockPopen(object):
     def __init__(self, stderr):
-        self.stdout = io.BytesIO('')
+        self.stdout = io.BytesIO(b'')
         self.stderr = io.BytesIO(stderr)
         self.returncode = 1
 
@@ -44,7 +46,7 b" INVALID_CERTIFICATE_STDERR = '\\n'.join(["
 @pytest.mark.xfail(sys.platform == "cygwin",
                    reason="SVN not packaged for Cygwin")
 def test_import_remote_repository_certificate_error(stderr, expected_reason):
-    from vcsserver import svn
+    from vcsserver.remote import svn
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
@@ -52,31 +54,29 b' def test_import_remote_repository_certif'
     remote.is_path_valid_repository = lambda wire, path: True
 
     with mock.patch('subprocess.Popen',
-                    return_value=MockPopen(stderr)):
+                    return_value=MockPopen(ascii_bytes(stderr))):
         with pytest.raises(Exception) as excinfo:
             remote.import_remote_repository({'path': 'path'}, 'url')
 
-    expected_error_args = (
-        'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason),)
+    expected_error_args = 'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason)
 
-    assert excinfo.value.args == expected_error_args
+    assert excinfo.value.args[0] == expected_error_args
 
 
 def test_svn_libraries_can_be_imported():
-    import svn
     import svn.client
     assert svn.client is not None
 
 
 @pytest.mark.parametrize('example_url, parts', [
-    ('http://server.com', (None, None, 'http://server.com')),
-    ('http://[e-mail]', ('user', None, 'http://[e-mail]')),
+    ('http://server.com', ('', '', 'http://server.com')),
+    ('http://[e-mail]', ('user', '', 'http://[e-mail]')),
     ('http://user:[e-mail]', ('user', 'pass', 'http://user:[e-mail]')),
-    ('<script>', (None, None, '<script>')),
-    ('http://', (None, None, 'http://')),
+    ('<script>', ('', '', '<script>')),
+    ('http://', ('', '', 'http://')),
 ])
 def test_username_password_extraction_from_url(example_url, parts):
-    from vcsserver import svn
+    from vcsserver.remote import svn
 
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
@@ -85,3 +85,19 b' def test_username_password_extraction_fr'
     remote.is_path_valid_repository = lambda wire, path: True
 
     assert remote.get_url_and_credentials(example_url) == parts
+
+
[e-mail]('call_url', [
+    b'https://svn.code.sf.net/p/svnbook/source/trunk/',
+    b'https://[e-mail]/p/svnbook/source/trunk/',
+    b'https://marcink:[e-mail]/p/svnbook/source/trunk/',
+])
+def test_check_url(call_url):
+    from vcsserver.remote import svn
+    factory = mock.Mock()
+    factory.repo = mock.Mock(return_value=mock.Mock())
+
+    remote = svn.SvnRemote(factory)
+    remote.is_path_valid_repository = lambda wire, path: True
+    assert remote.check_url(call_url, {'dummy': 'config'})
+

vcsserver/tests/test_wsgi_app_caller.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,25 +19,26 b' import wsgiref.simple_server'
 import wsgiref.validate
 
 from vcsserver import wsgi_app_caller
-
-
-# pylint: disable=protected-access,too-many-public-methods
+from vcsserver.str_utils import ascii_bytes, safe_str
 
 
 @wsgiref.validate.validator
 def demo_app(environ, start_response):
     """WSGI app used for testing."""
+
+    input_data = safe_str(environ['wsgi.input'].read(1024))
+
     data = [
         'Hello World!\n',
-        'input_data=%s\n' % environ['wsgi.input'].read(),
+        f'input_data={input_data}\n',
     ]
     for key, value in sorted(environ.items()):
-        data.append('%s=%s\n' % (key, value))
+        data.append(f'{key}={value}\n')
 
     write = start_response("200 OK", [('Content-Type', 'text/plain')])
-    write('Old school write method\n')
-    write('***********************\n')
-    return data
+    write(b'Old school write method\n')
+    write(b'***********************\n')
+    return list(map(ascii_bytes, data))
 
 
 BASE_ENVIRON = {
@@ -53,11 +54,11 b' BASE_ENVIRON = {'
 
 def test_complete_environ():
     environ = dict(BASE_ENVIRON)
-    data = "data"
+    data = b"data"
     wsgi_app_caller._complete_environ(environ, data)
     wsgiref.validate.check_environ(environ)
 
-    assert data == environ['wsgi.input'].read()
+    assert data == environ['wsgi.input'].read(1024)
 
 
 def test_start_response():
@@ -81,16 +82,17 b' def test_start_response_with_error():'
 
 
 def test_wsgi_app_caller():
-    caller = wsgi_app_caller.WSGIAppCaller(demo_app)
     environ = dict(BASE_ENVIRON)
     input_data = 'some text'
+
+    caller = wsgi_app_caller.WSGIAppCaller(demo_app)
     responses, status, headers = caller.handle(environ, input_data)
-    response = ''.join(responses)
+    response = b''.join(responses)
 
     assert status == '200 OK'
     assert headers == [('Content-Type', 'text/plain')]
-    assert response.startswith(
-        'Old school write method\n***********************\n')
-    assert 'Hello World!\n' in response
-    assert 'foo.var=bla\n' in response
-    assert 'input_data=%s\n' % input_data in response
+    assert response.startswith(b'Old school write method\n***********************\n')
+    assert b'Hello World!\n' in response
+    assert b'foo.var=bla\n' in response
+
+    assert ascii_bytes(f'input_data={input_data}\n') in response

vcsserver/tweens/__init__.py

@@ -1,6 +1,4 b''
-# -*- coding: utf-8 -*-
-
-# Copyright (C) 2016-2020 RhodeCode GmbH
+# Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3

vcsserver/tweens/request_wrapper.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,13 +14,14 b''
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
-
+import base64
 import time
 import logging
 
+import msgpack
+
 import vcsserver
-from vcsserver.utils import safe_str
-
+from vcsserver.str_utils import safe_str, ascii_str
 
 log = logging.getLogger(__name__)
 
@@ -34,12 +35,24 b' def get_user_agent(environ):'
     return environ.get('HTTP_USER_AGENT')
 
 
-def get_vcs_method(environ):
-    return environ.get('HTTP_X_RC_METHOD')
+def get_call_context(request) -> dict:
+    cc = {}
+    registry = request.registry
+    if hasattr(registry, 'vcs_call_context'):
+        cc.update({
+            'X-RC-Method': registry.vcs_call_context.get('method'),
+            'X-RC-Repo-Name': registry.vcs_call_context.get('repo_name')
+        })
+
+    return cc
 
 
-def get_vcs_repo(environ):
-    return environ.get('HTTP_X_RC_REPO_NAME')
+def get_headers_call_context(environ, strict=True):
+    if 'HTTP_X_RC_VCS_STREAM_CALL_CONTEXT' in environ:
+        packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
+        return msgpack.unpackb(packed_cc)
+    elif strict:
+        raise ValueError('Expected header HTTP_X_RC_VCS_STREAM_CALL_CONTEXT not found')
 
 
 class RequestWrapperTween(object):
@@ -54,23 +67,26 b' class RequestWrapperTween(object):'
         log.debug('Starting request time measurement')
         response = None
 
-        ua = get_user_agent(request.environ)
-        vcs_method = get_vcs_method(request.environ)
-        repo_name = get_vcs_repo(request.environ)
-
         try:
             response = self.handler(request)
         finally:
+            ua = get_user_agent(request.environ)
+            call_context = get_call_context(request)
+            vcs_method = call_context.get('X-RC-Method', '_NO_VCS_METHOD')
+            repo_name = call_context.get('X-RC-Repo-Name', '')
+
             count = request.request_count()
             _ver_ = vcsserver.__version__
             _path = safe_str(get_access_path(request.environ))
+
             ip = '127.0.0.1'
             match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
             resp_code = getattr(response, 'status_code', 'UNDEFINED')
 
+            _view_path = f"{repo_name}@{_path}/{vcs_method}"
+
             total = time.time() - start
 
-            _view_path = "{}/{}@{}".format(_path, vcs_method, repo_name)
             log.info(
                 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
                 count, ip, request.environ.get('REQUEST_METHOD'),
@@ -87,15 +103,15 b' class RequestWrapperTween(object):'
                 statsd.timing(
                     "vcsserver_req_timing.histogram", elapsed_time_ms,
                     tags=[
-                        "view_name:{}".format(match_route),
-                        "code:{}".format(resp_code)
+                        f"view_name:{match_route}",
+                        f"code:{resp_code}"
                     ],
                     use_decimals=False
                 )
                 statsd.incr(
                     "vcsserver_req_total", tags=[
-                        "view_name:{}".format(match_route),
-                        "code:{}".format(resp_code)
+                        f"view_name:{match_route}",
+                        f"code:{resp_code}"
                     ])
 
         return response

vcsserver/utils.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,91 +20,35 b' import hashlib'
 log = logging.getLogger(__name__)
 
 
-def safe_int(val, default=None):
-    """
-    Returns int() of val if val is not convertable to int use default
-    instead
-
-    :param val:
-    :param default:
-    """
-
-    try:
-        val = int(val)
-    except (ValueError, TypeError):
-        val = default
-
-    return val
-
-
-def safe_str(unicode_, to_encoding=None):
-    """
-    safe str function. Does few trick to turn unicode_ into string
-
-    :param unicode_: unicode to encode
-    :param to_encoding: encode to this type UTF8 default
-    :rtype: str
-    :returns: str object
-    """
-    to_encoding = to_encoding or ['utf8']
-    # if it's not basestr cast to str
-    if not isinstance(unicode_, basestring):
-        return str(unicode_)
-
-    if isinstance(unicode_, str):
-        return unicode_
-
-    if not isinstance(to_encoding, (list, tuple)):
-        to_encoding = [to_encoding]
+class AttributeDictBase(dict):
+    def __getstate__(self):
+        odict = self.__dict__  # get attribute dictionary
+        return odict
 
-    for enc in to_encoding:
-        try:
-            return unicode_.encode(enc)
-        except UnicodeEncodeError:
-            pass
-
-    return unicode_.encode(to_encoding[0], 'replace')
-
-
-def safe_unicode(str_, from_encoding=None):
-    """
-    safe unicode function. Does few trick to turn str_ into unicode
-
-    :param str_: string to decode
-    :param from_encoding: encode from this type UTF8 default
-    :rtype: unicode
-    :returns: unicode object
-    """
-    from_encoding = from_encoding or ['utf8']
+    def __setstate__(self, dict):
+        self.__dict__ = dict
 
-    if isinstance(str_, unicode):
-        return str_
-
-    if not isinstance(from_encoding, (list, tuple)):
-        from_encoding = [from_encoding]
-
-    try:
-        return unicode(str_)
-    except UnicodeDecodeError:
-        pass
-
-    for enc in from_encoding:
-        try:
-            return unicode(str_, enc)
-        except UnicodeDecodeError:
-            pass
-
-    return unicode(str_, from_encoding[0], 'replace')
-
-
-class AttributeDict(dict):
-    def __getattr__(self, attr):
-        return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
+class StrictAttributeDict(AttributeDictBase):
+    """
+    Strict Version of Attribute dict which raises an Attribute error when
+    requested attribute is not set
+    """
+    def __getattr__(self, attr):
+        try:
+            return self[attr]
+        except KeyError:
+            raise AttributeError('{} object has no attribute {}'.format(
+                self.__class__, attr))
+
+
+class AttributeDict(AttributeDictBase):
+    def __getattr__(self, attr):
+        return self.get(attr, None)
+
+
 def sha1(val):
     return hashlib.sha1(val).hexdigest()
-
-

vcsserver/vcs_base.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,17 +17,18 b''
 
 from vcsserver.lib import rc_cache
 
+
 class RemoteBase(object):
     EMPTY_COMMIT = '0' * 40
 
     def _region(self, wire):
         cache_repo_id = wire.get('cache_repo_id', '')
-        cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
+        cache_namespace_uid = f'cache_repo.{cache_repo_id}'
         return rc_cache.get_or_create_region('repo_object', cache_namespace_uid)
 
     def _cache_on(self, wire):
         context = wire.get('context', '')
-        context_uid = '{}'.format(context)
+        context_uid = f'{context}'
         repo_id = wire.get('repo_id', '')
         cache = wire.get('cache', True)
         cache_on = context and cache
@@ -37,10 +38,10 b' class RemoteBase(object):'
         from vcsserver.lib import rc_cache
         repo_id = wire.get('repo_id', '')
         cache_repo_id = wire.get('cache_repo_id', '')
-        cache_namespace_uid = 'cache_repo.{}'.format(cache_repo_id)
+        cache_namespace_uid = f'cache_repo.{cache_repo_id}'
 
         if delete:
             rc_cache.clear_cache_namespace(
-                'repo_object', cache_namespace_uid, invalidate=True)
+                'repo_object', cache_namespace_uid, method=rc_cache.CLEAR_DELETE)
 
         return {'invalidated': {'repo_id': repo_id, 'delete': delete}}

vcsserver/wsgi_app_caller.py

@@ -1,5 +1,5 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2020 RhodeCode GmbH
+# Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,19 +23,20 b' import io'
 import logging
 import os
 
+from vcsserver.str_utils import ascii_bytes
 
 log = logging.getLogger(__name__)
 
 DEV_NULL = open(os.devnull)
 
 
-def _complete_environ(environ, input_data):
+def _complete_environ(environ, input_data: bytes):
     """Update the missing wsgi.* variables of a WSGI environment.
 
     :param environ: WSGI environment to update
     :type environ: dict
     :param input_data: data to be read by the app
-    :type input_data: str
+    :type input_data: bytes
     """
     environ.update({
         'wsgi.version': (1, 0),
@@ -92,20 +93,19 b' class WSGIAppCaller(object):'
         :param environ: WSGI environment to update
         :type environ: dict
         :param input_data: data to be read by the app
-        :type input_data: str
+        :type input_data: str/bytes
 
         :returns: a tuple with the contents, status and headers
         :rtype: (list<str>, str, list<(str, str)>)
         """
-        _complete_environ(environ, input_data)
+        _complete_environ(environ, ascii_bytes(input_data, allow_bytes=True))
         start_response = _StartResponse()
         log.debug("Calling wrapped WSGI application")
         responses = self.app(environ, start_response)
         responses_list = list(responses)
         existing_responses = start_response.content
         if existing_responses:
-            log.debug(
-                "Adding returned response to response written via write()")
+            log.debug("Adding returned response to response written via write()")
             existing_responses.extend(responses_list)
             responses_list = existing_responses
         if hasattr(responses, 'close'):