core: multiple changes for python3 found during test runs of rhodecode-ce release

vcsserver/hooks.py

@@ -586,9 +586,9 b' def git_post_receive(unused_repo_path, r'
                 try:
                     subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
-                    cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"',
-                           '"refs/heads/%s"' % push_ref['name']]
-                    print(("Setting default branch to %s" % push_ref['name']))
+                    push_ref_name = push_ref['name']
+                    cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"', f'"refs/heads/{push_ref_name}"']
+                    print(f"Setting default branch to {push_ref_name}")
                     subprocessio.run_command(cmd, env=os.environ.copy())
 
                 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',

vcsserver/remote/git.py

@@ -40,7 +40,7 b' from dulwich.repo import Repo as Dulwich'
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
-from vcsserver.str_utils import safe_str, safe_int, safe_bytes
+from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_str, ascii_bytes
 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
@@ -51,7 +51,7 b' DIR_STAT = stat.S_IFDIR'
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 PEELED_REF_MARKER = b'^{}'
-
+HEAD_MARKER = b'HEAD'
 
 log = logging.getLogger(__name__)
 
@@ -107,8 +107,9 b' class GitFactory(RepoFactory):'
 
     def _create_repo(self, wire, create, use_libgit2=False):
         if use_libgit2:
-            return Repository(wire['path'])
+            return Repository(safe_bytes(wire['path']))
         else:
+            # dulwich mode
             repo_path = safe_str(wire['path'], to_encoding=settings.WIRE_ENCODING)
             return Repo(repo_path)
 
@@ -610,13 +611,25 b' class GitRemote(RemoteBase):'
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
-            def determine_wants_requested(references):
-                return [references[r] for r in references if r in refs]
+            refs = [ascii_bytes(x) for x in refs]
+
+            def determine_wants_requested(remote_refs):
+                determined = []
+                for ref_name, ref_hash in remote_refs.items():
+                    bytes_ref_name = safe_bytes(ref_name)
+
+                    if bytes_ref_name in refs:
+                        bytes_ref_hash = safe_bytes(ref_hash)
+                        determined.append(bytes_ref_hash)
+                return determined
+
+            # swap with our custom requested wants
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
+
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
@@ -641,13 +654,13 b' class GitRemote(RemoteBase):'
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
-                repo["HEAD"] = remote_refs[refs[-1]]
+                repo[HEAD_MARKER] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to checkout HEAD
-            repo["HEAD"] = remote_refs["HEAD"]
+            repo[HEAD_MARKER] = remote_refs[HEAD_MARKER]
             index.build_index_from_tree(repo.path, repo.index_path(),
-                                        repo.object_store, repo["HEAD"].tree)
+                                        repo.object_store, repo[HEAD_MARKER].tree)
         return remote_refs
 
     @reraise_safe_exceptions
@@ -680,7 +693,7 b' class GitRemote(RemoteBase):'
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
-            if ref in [b'HEAD']:
+            if ref in [HEAD_MARKER]:
                 continue
 
             remote_refs[ref] = sha
@@ -1292,6 +1305,10 b' class GitRemote(RemoteBase):'
         from vcsserver.hook_utils import install_git_hooks
         bare = self.bare(wire)
         path = wire['path']
+        binary_dir = settings.BINARY_DIR
+        executable = None
+        if binary_dir:
+            executable = os.path.join(binary_dir, 'python3')
         return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions

vcsserver/remote/hg.py

@@ -152,7 +152,7 b' class MercurialFactory(RepoFactory):'
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
-        return instance(baseui, ascii_bytes(wire["path"]), create)
+        return instance(baseui, safe_bytes(wire["path"]), create)
 
     def repo(self, wire, create=False):
         """
@@ -757,12 +757,13 b' class HgRemote(RemoteBase):'
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
-        other_path = kwargs.pop('other_path', None)
+        org_path = safe_bytes(wire["path"])
+        other_path = safe_bytes(kwargs.pop('other_path', ''))
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
-            repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
+            repo = unionrepo.makeunionrepository(baseui, other_path, org_path)
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
@@ -815,9 +816,11 b' class HgRemote(RemoteBase):'
         return _tags(context_uid, repo_id)
 
     @reraise_safe_exceptions
-    def update(self, wire, node=None, clean=False):
+    def update(self, wire, node='', clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
+        node = safe_bytes(node)
+
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
@@ -855,7 +858,7 b' class HgRemote(RemoteBase):'
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
-        a = changelog.ancestor(lookup(revision1), lookup(revision2))
+        a = changelog.ancestor(lookup(safe_bytes(revision1)), lookup(safe_bytes(revision2)))
         return hex(a)
 
     @reraise_safe_exceptions
@@ -933,19 +936,29 b' class HgRemote(RemoteBase):'
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
-    def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
+    def pull_cmd(self, wire, source, bookmark='', branch='', revision='', hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
+        source = safe_bytes(source)
+
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
         if bookmark:
+            if isinstance(branch, list):
+                bookmark = [safe_bytes(x) for x in bookmark]
+            else:
+                bookmark = safe_bytes(bookmark)
             opts['bookmark'] = bookmark
         if branch:
+            if isinstance(branch, list):
+                branch = [safe_bytes(x) for x in branch]
+            else:
+                branch = safe_bytes(branch)
             opts['branch'] = branch
         if revision:
-            opts['rev'] = revision
+            opts['rev'] = safe_bytes(revision)
 
         commands.pull(baseui, repo, source, **opts)
 
@@ -1040,10 +1053,10 b' class HgRemote(RemoteBase):'
             raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
-    def bookmark(self, wire, bookmark, revision=None):
+    def bookmark(self, wire, bookmark, revision=''):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
-        commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
+        commands.bookmark(baseui, repo, safe_bytes(bookmark), rev=safe_bytes(revision), force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):

vcsserver/remote/svn.py

@@ -38,7 +38,7 b' import svn.repos'
 from vcsserver import svn_diff, exceptions, subprocessio, settings
 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
 from vcsserver.exceptions import NoContentException
-from vcsserver.str_utils import safe_str
+from vcsserver.str_utils import safe_str, safe_bytes
 from vcsserver.vcs_base import RemoteBase
 from vcsserver.lib.svnremoterepo import svnremoterepo
 log = logging.getLogger(__name__)
@@ -63,7 +63,7 b' def reraise_safe_exceptions(func):'
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in svn remote call")
-                raise_from_original(exceptions.UnhandledException(e))
+                raise_from_original(exceptions.UnhandledException(e), e)
             raise
     return wrapper
 
@@ -328,6 +328,7 b' class SvnRemote(RemoteBase):'
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
+
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_nodes(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
@@ -425,8 +426,11 b' class SvnRemote(RemoteBase):'
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
-        assert isinstance(message, str)
-        assert isinstance(author, str)
+
+        updated = [{k: safe_bytes(v) for k, v in x.items() if isinstance(v, str)} for x in updated]
+
+        message = safe_bytes(message)
+        author = safe_bytes(author)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
@@ -477,7 +481,7 b' class SvnRemote(RemoteBase):'
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _rev, _path):
             raw_bytes = self.get_file_content(wire, path, rev)
-            return raw_bytes and '\0' in raw_bytes
+            return raw_bytes and b'\0' in raw_bytes
 
         return _is_binary(repo_id, rev, path)
 
@@ -530,9 +534,8 b' class SvnRemote(RemoteBase):'
         binary_dir = settings.BINARY_DIR
         executable = None
         if binary_dir:
-            executable = os.path.join(binary_dir, 'python')
-        return install_svn_hooks(
-            repo_path, executable=executable, force_create=force)
+            executable = os.path.join(binary_dir, 'python3')
+        return install_svn_hooks(repo_path, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
@@ -808,7 +811,7 b' class TxnNodeProcessor(object):'
     """
 
     def __init__(self, node, txn_root):
-        assert isinstance(node['path'], str)
+        assert isinstance(node['path'], bytes)
 
         self.node = node
         self.txn_root = txn_root
@@ -844,7 +847,8 b' class TxnNodeProcessor(object):'
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
-        assert isinstance(self.node['content'], str)
+        assert isinstance(self.node['content'], bytes)
+
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)

vcsserver/tests/conftest.py

@@ -39,7 +39,7 b' def repeat(request):'
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = get_available_port()
-    print(('Using vcsserver port %s' % (port, )))
+    print(f'Using vcsserver port {port}')
     return port
 
 

vcsserver/tests/test_git.py

@@ -66,8 +66,8 b' class TestGitFetch(object):'
 
     def test_fetches_specified_commits(self):
         selected_refs = {
-            'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
-            'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
+            'refs/tags/v0.1.8': b'74ebce002c088b8a5ecf40073db09375515ecd68',
+            'refs/tags/v0.1.3': b'5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         def side_effect(determine_wants, *args, **kwargs):