git: bumped to latest git version.

- security release
- fixed patches
- added setreference quotes

pkgs/overlays.nix

@@ -1,83 +1,83 b''
 self: super: {
 
   # bump GIT version
   git =
   let
     gitWithoutPerl = super.git.override {
       #perlSupport = false;
     };
   in
    super.lib.overrideDerivation gitWithoutPerl (oldAttrs: {
 
-    name = "git-2.30.0";
+    name = "git-2.39.1";
     src = self.fetchurl {
-      url = "https://www.kernel.org/pub/software/scm/git/git-2.30.0.tar.xz";
-      sha256 = "06ad6dylgla34k9am7d5z8y3rryc8ln3ibq5z0d74rcm20hm0wsm";
+      url = "https://www.kernel.org/pub/software/scm/git/git-2.39.1.tar.xz";
+      sha256 = "0qf1wly7zagg23svpv533va5v213y7y3lfw76ldkf35k8w48m8s0";
     };
 
-    # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
+    # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git
     patches = [
       ./patches/git/docbook2texi.patch
+      ./patches/git/git-send-email-honor-PATH.patch
       ./patches/git/git-sh-i18n.patch
+      ./patches/git/installCheck-path.patch
       ./patches/git/ssh-path.patch
-      ./patches/git/git-send-email-honor-PATH.patch
-      ./patches/git/installCheck-path.patch
     ];
 
     #preInstallCheck = oldAttrs.preInstallCheck + ''
     #    disable_test t4129-apply-samemode
     #    disable_test t5324-split-commit-graph
     #'';
 
   });
 
   libgit2rc = super.lib.overrideDerivation super.libgit2 (oldAttrs: {
     name = "libgit2-0.28.2";
     version = "0.28.2";
 
     src = self.fetchFromGitHub {
         owner = "libgit2";
         repo = "libgit2";
         rev = "v0.28.2";
         sha256 = "0cm8fvs05rj0baigs2133q5a0sm3pa234y8h6hmwhl2bz9xq3k4b";
     };
 
     cmakeFlags = [ "-DTHREADSAFE=ON" "-DUSE_HTTPS=no"];
 
     buildInputs = [
         super.zlib
         super.libssh2
         super.openssl
         super.curl
     ];
 
 
   });
 
   # Override subversion derivation to
   #  - activate python bindings
   subversion =
   let
     subversionWithPython = super.subversion.override {
       httpSupport = true;
       pythonBindings = true;
       python = self.python27Packages.python;
     };
   in
     super.lib.overrideDerivation subversionWithPython (oldAttrs: {
       name = "subversion-1.13.0";
       src = self.fetchurl {
         url = "https://archive.apache.org/dist/subversion/subversion-1.13.0.tar.gz";
         sha256 = "0cb9p7f5hg0l4k32hz8vmvy2r45igchq5sh4m366za5q0c649bfs";
       };
 
       ## use internal lz4/utf8proc because it is stable and shipped with SVN
       configureFlags = oldAttrs.configureFlags ++ [
         " --with-lz4=internal"
         " --with-utf8proc=internal"
       ];
 
   });
 
 
 }

pkgs/patches/git/docbook2texi.patch

@@ -1,38 +1,38 b''
 This patch does two things: (1) use the right name for `docbook2texi',
 and (2) make sure `gitman.info' isn't produced since it's broken (duplicate
 node names).
 
 diff --git a/Documentation/Makefile b/Documentation/Makefile
 index 26a2342bea..ceccd67ebb 100644
 --- a/Documentation/Makefile
 +++ b/Documentation/Makefile
 @@ -132,7 +132,7 @@ HTML_REPO = ../../git-htmldocs
 
  MAKEINFO = makeinfo
  INSTALL_INFO = install-info
 -DOCBOOK2X_TEXI = docbook2x-texi
 +DOCBOOK2X_TEXI = docbook2texi
  DBLATEX = dblatex
  ASCIIDOC_DBLATEX_DIR = /etc/asciidoc/dblatex
  DBLATEX_COMMON = -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty
 @@ -250,7 +250,7 @@ man1: $(DOC_MAN1)
  man5: $(DOC_MAN5)
  man7: $(DOC_MAN7)
 
 -info: git.info gitman.info
 +info: git.info
 
  pdf: user-manual.pdf
 
 @@ -266,10 +266,9 @@ install-man: man
 
  install-info: info
  	$(INSTALL) -d -m 755 $(DESTDIR)$(infodir)
 -	$(INSTALL) -m 644 git.info gitman.info $(DESTDIR)$(infodir)
 +	$(INSTALL) -m 644 git.info $(DESTDIR)$(infodir)
  	if test -r $(DESTDIR)$(infodir)/dir; then \
  	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) git.info ;\
 -	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) gitman.info ;\
  	else \
  	  echo "No directory found in $(DESTDIR)$(infodir)" >&2 ; \
- 	fi
+ 	fi
  No newline at end of file

pkgs/patches/git/git-send-email-honor-PATH.patch

@@ -1,28 +1,31 b''
 diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt
-index 1afe9fc858..05dd7c3a90 100644
+index 3db4eab4ba..39bc0e77c9 100644
 --- a/Documentation/git-send-email.txt
 +++ b/Documentation/git-send-email.txt
-@@ -215,8 +215,7 @@ a password is obtained using 'git-credential'.
- 	specify a full pathname of a sendmail-like program instead;
- 	the program must support the `-i` option.  Default value can
- 	be specified by the `sendemail.smtpServer` configuration
--	option; the built-in default is to search for `sendmail` in
--	`/usr/sbin`, `/usr/lib` and $PATH if such program is
-+	option; the built-in default is to search in $PATH if such program is
- 	available, falling back to `localhost` otherwise.
-
- --smtp-server-port=<port>::
+@@ -220,9 +220,9 @@ a password is obtained using 'git-credential'.
+ --smtp-server=<host>::
+ 	If set, specifies the outgoing SMTP server to use (e.g.
+ 	`smtp.example.com` or a raw IP address).  If unspecified, and if
+-	`--sendmail-cmd` is also unspecified, the default is to search
+-	for `sendmail` in `/usr/sbin`, `/usr/lib` and $PATH if such a
+-	program is available, falling back to `localhost` otherwise.
++	`--sendmail-cmd` is also unspecified, the default is to search for
++	`sendmail` in $PATH if such a program is available, falling back to
++	`localhost` otherwise.
+ +
+ For backward compatibility, this option can also specify a full pathname
+ of a sendmail-like program instead; the program must support the `-i`
 diff --git a/git-send-email.perl b/git-send-email.perl
-index 8eb63b5a2f..74a61d8213 100755
+index e65d969d0b..508d49483d 100755
 --- a/git-send-email.perl
 +++ b/git-send-email.perl
-@@ -956,8 +956,7 @@ sub expand_one_alias {
+@@ -1066,8 +1066,7 @@ sub expand_one_alias {
  }
 
- if (!defined $smtp_server) {
+ if (!defined $sendmail_cmd && !defined $smtp_server) {
 -	my @sendmail_paths = qw( /usr/sbin/sendmail /usr/lib/sendmail );
 -	push @sendmail_paths, map {"$_/sendmail"} split /:/, $ENV{PATH};
 +	my @sendmail_paths = map {"$_/sendmail"} split /:/, $ENV{PATH};
  	foreach (@sendmail_paths) {
  		if (-x $_) {
- 			$smtp_server = $_;
+ 			$sendmail_cmd = $_;
  No newline at end of file

pkgs/patches/git/git-sh-i18n.patch

@@ -1,23 +1,23 b''
 diff --git a/git-sh-i18n.sh b/git-sh-i18n.sh
 index e1d917fd27..e90f8e1414 100644
 --- a/git-sh-i18n.sh
 +++ b/git-sh-i18n.sh
 @@ -26,7 +26,7 @@ then
  elif test -n "$GIT_INTERNAL_GETTEXT_TEST_FALLBACKS"
  then
  	: no probing necessary
 -elif type gettext.sh >/dev/null 2>&1
 +elif type @gettext@/bin/gettext.sh >/dev/null 2>&1
  then
  	# GNU libintl's gettext.sh
  	GIT_INTERNAL_GETTEXT_SH_SCHEME=gnu
 @@ -43,7 +43,8 @@ export GIT_INTERNAL_GETTEXT_SH_SCHEME
  case "$GIT_INTERNAL_GETTEXT_SH_SCHEME" in
  gnu)
  	# Use libintl's gettext.sh, or fall back to English if we can't.
 -	. gettext.sh
 +	. @gettext@/bin/gettext.sh
 +	export PATH=@gettext@/bin:$PATH
  	;;
  gettext_without_eval_gettext)
- 	# Solaris has a gettext(1) but no eval_gettext(1)
+ 	# Solaris has a gettext(1) but no eval_gettext(1)
  No newline at end of file

pkgs/patches/git/installCheck-path.patch

@@ -1,13 +1,13 b''
 diff --git a/t/test-lib.sh b/t/test-lib.sh
 index 8665b0a9b6..8bb892b1af 100644
 --- a/t/test-lib.sh
 +++ b/t/test-lib.sh
 @@ -1227,7 +1227,7 @@ elif test -n "$GIT_TEST_INSTALLED"
  then
  	GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path)  ||
  	error "Cannot run git from $GIT_TEST_INSTALLED."
 -	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH
 +	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$GIT_BUILD_DIR:$PATH
  	GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
  else # normal case, use ../bin-wrappers only unless $with_dashes:
- 	if test -n "$no_bin_wrappers"
+ 	if test -n "$no_bin_wrappers"
  No newline at end of file

pkgs/patches/git/ssh-path.patch

@@ -1,26 +1,26 b''
 diff --git a/connect.c b/connect.c
 index 4813f005ab..b3f12f3268 100644
 --- a/connect.c
 +++ b/connect.c
 @@ -1183,7 +1183,7 @@ static void fill_ssh_args(struct child_process *conn, const char *ssh_host,
 
  		ssh = getenv("GIT_SSH");
  		if (!ssh)
 -			ssh = "ssh";
 +			ssh = "@ssh@";
  		variant = determine_ssh_variant(ssh, 0);
  	}
 
 diff --git a/git-gui/lib/remote_add.tcl b/git-gui/lib/remote_add.tcl
 index 480a6b30d0..7817204241 100644
 --- a/git-gui/lib/remote_add.tcl
 +++ b/git-gui/lib/remote_add.tcl
 @@ -139,7 +139,7 @@ method _add {} {
  		# Parse the location
  		if { [regexp {(?:git\+)?ssh://([^/]+)(/.+)} $location xx host path]
  		     || [regexp {([^:][^:]+):(.+)} $location xx host path]} {
 -			set ssh ssh
 +			set ssh @ssh@
  			if {[info exists env(GIT_SSH)]} {
  				set ssh $env(GIT_SSH)
- 			}
+ 			}
  No newline at end of file

vcsserver/git.py

@@ -1,1281 +1,1281 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import collections
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import traceback
 import urllib
 import urllib2
 from functools import wraps
 
 import more_itertools
 import pygit2
 from pygit2 import Repository as LibGit2Repo
 from pygit2 import index as LibGit2Index
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str, safe_int, safe_unicode
 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
 from vcsserver.vcs_base import RemoteBase
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 PEELED_REF_MARKER = '^{}'
 
 
 log = logging.getLogger(__name__)
 
 
 def str_to_dulwich(value):
     """
     Dulwich 0.10.1a requires `unicode` objects to be passed in.
     """
     return value.decode(settings.WIRE_ENCODING)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
 
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
             exc = exceptions.LookupException(org_exc=e)
             raise exc(safe_str(e))
         except (HangupException, UnexpectedCommandError) as e:
             exc = exceptions.VcsException(org_exc=e)
             raise exc(safe_str(e))
         except Exception as e:
             # NOTE(marcink): becuase of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
                 #log.exception("Unhandled exception in git remote call")
                 #raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class Repository(LibGit2Repo):
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.free()
 
 
 class GitFactory(RepoFactory):
     repo_type = 'git'
 
     def _create_repo(self, wire, create, use_libgit2=False):
         if use_libgit2:
             return Repository(wire['path'])
         else:
             repo_path = str_to_dulwich(wire['path'])
             return Repo(repo_path)
 
     def repo(self, wire, create=False, use_libgit2=False):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create, use_libgit2)
 
     def repo_libgit2(self, wire):
         return self.repo(wire, use_libgit2=True)
 
 
 class GitRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
         self._bulk_methods = {
             "date": self.date,
             "author": self.author,
             "branch": self.branch,
             "message": self.message,
             "parents": self.parents,
             "_commit": self.revision,
         }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
             return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
         return {}
 
     def _remote_conf(self, config):
         params = [
             '-c', 'core.askpass=""',
         ]
         ssl_cert_dir = config.get('vcs_ssl_dir')
         if ssl_cert_dir:
             params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
         return params
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = 'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return stdout.strip()
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
 
             try:
                 has_head = repo.head.name
                 if has_head:
                     return False
 
                 # NOTE(marcink): check again using more expensive method
                 return repo.is_empty
             except Exception:
                 pass
 
             return True
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id):
             try:
                 repo_init = self._factory.repo_libgit2(wire)
                 with repo_init as repo:
                     pass
             except pygit2.GitError:
                 path = wire.get('path')
                 tb = traceback.format_exc()
                 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
                 return False
 
             return True
         return _assert_correct_path(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             return repo.is_bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob_obj = repo[sha]
             blob = blob_obj.data
             return blob
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _blob_raw_length(_repo_id, _sha):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[sha]
                 return blob.size
 
         return _blob_raw_length(repo_id, sha)
 
     def _parse_lfs_pointer(self, raw_content):
 
         spec_string = 'version https://git-lfs.github.com/spec'
         if raw_content and raw_content.startswith(spec_string):
             pattern = re.compile(r"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
             ^size[ ](?P<oid_size>[0-9]+)\n
             (?:\n)?
             """, re.VERBOSE | re.MULTILINE)
             match = pattern.match(raw_content)
             if match:
                 return match.groupdict()
 
         return {}
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[commit_id]
                 if blob.is_binary:
                     return {}
 
                 return self._parse_lfs_pointer(blob.data)
 
         return _is_large_file(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _tree_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob_obj = repo[tree_id]
                 return blob_obj.is_binary
 
         return _is_binary(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
 
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.has_oid()
 
         return False
 
     @reraise_safe_exceptions
     def store_path(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
         raise ValueError('Unable to fetch oid with path {}'.format(oid))
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _rev, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
                     args = [wire, rev]
                     result[attr] = method(*args)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(
                         "Unknown bulk attribute: %s" % attr)
             return result
 
         return _bulk_request(repo_id, rev, sorted(pre_load))
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
     def _type_id_to_name(self, type_id):
         return {
             1: b'commit',
             2: b'tree',
             3: b'blob',
             4: b'tag'
         }[type_id]
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError()(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
         remote_refs = self.pull(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     @reraise_safe_exceptions
     def branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _branch(_context_uid, _repo_id, _commit_id):
             regex = re.compile('^refs/heads')
 
             def filter_with(ref):
                 return regex.match(ref[0]) and ref[1] == _commit_id
 
             branches = filter(filter_with, self.get_refs(wire).items())
             return [x[0].split('refs/heads/')[-1] for x in branches]
 
         return _branch(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def commit_branches(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _commit_branches(_context_uid, _repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 branches = [x for x in repo.branches.with_commit(_commit_id)]
                 return branches
 
         return _commit_branches(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob = objects.Blob()
             blob.set_raw_string(content)
             repo.object_store.add_object(blob)
             return blob.id
 
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         # Defines the root tree
         class _Root(object):
             def __repr__(self):
                 return 'ROOT TREE'
         ROOT = _Root()
 
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
 
         if commit_tree and repo[commit_tree]:
             git_commit = repo[commit_data['parents'][0]]
             commit_tree = repo[git_commit.tree]  # root tree
         else:
             commit_tree = objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
                 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree  # start with top-level
             trees = [{'tree': tree, 'path': ROOT}]
             # Traverse deep into the forest...
             # resolve final tree by iterating the path.
             # e.g a/b/c.txt will get
             # - root as tree then
             # - 'a' as tree,
             # - 'b' as tree,
             # - stop at c as blob.
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append({'tree': obj, 'path': path})
                         tree = obj
                 except KeyError:
                     break
             #PROBLEM:
             """
             We're not editing same reference tree object
             """
             # Cut down the blob and all rotten trees on the way back...
             for path, tree_data in reversed(zip(paths, trees)):
                 tree = tree_data['tree']
                 tree.__delitem__(path)
                 # This operation edits the tree, we need to mark new commit back
 
                 if len(tree) > 0:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.items():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
         self.create_branch(wire, branch, commit.id)
 
         # dulwich set-ref
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             for k in remote_refs:
                 if k.endswith(PEELED_REF_MARKER):
                     log.debug("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
                 repo["HEAD"] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_fetch(self, wire, url, refs=None, all_refs=False):
         repo = self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
 
         config = self._wire_to_config(wire)
         # get all remote refs we'll use to fetch later
         cmd = ['ls-remote']
         if not all_refs:
             cmd += ['--heads', '--tags']
         cmd += [url]
         output, __ = self.run_git_command(
             wire, cmd, fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         remote_refs = collections.OrderedDict()
         fetch_refs = []
 
         for ref_line in output.splitlines():
             sha, ref = ref_line.split('\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
                 continue
             if ref.endswith(PEELED_REF_MARKER):
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
             if ref in ['HEAD']:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
                 fetch_refs.append('{}:{}'.format(ref, ref))
             elif not refs:
                 fetch_refs.append('{}:{}'.format(ref, ref))
         log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
 
         if fetch_refs:
             for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
                 fetch_refs_chunks = list(chunk)
                 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
                 _out, _err = self.run_git_command(
                     wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
                     extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url, refs=None):
         if not self.check_url(url, wire):
             return
         config = self._wire_to_config(wire)
         self._factory.repo(wire)
         self.run_git_command(
             wire, ['push', url, '--mirror'], fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha, maybe_unreachable=False):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_object(_context_uid, _repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
 
                 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
                 try:
                     commit = repo.revparse_single(sha)
                 except KeyError:
                     # NOTE(marcink): KeyError doesn't give us any meaningful information
                     # here, we instead give something more explicit
                     e = exceptions.RefNotFoundException('SHA: %s not found', sha)
                     raise exceptions.LookupException(e)(missing_commit_err)
                 except ValueError as e:
                     raise exceptions.LookupException(e)(missing_commit_err)
 
                 is_tag = False
                 if isinstance(commit, pygit2.Tag):
                     commit = repo.get(commit.target)
                     is_tag = True
 
                 check_dangling = True
                 if is_tag:
                     check_dangling = False
 
                 if check_dangling and maybe_unreachable:
                     check_dangling = False
 
                 # we used a reference and it parsed means we're not having a dangling commit
                 if sha != commit.hex:
                     check_dangling = False
 
                 if check_dangling:
                     # check for dangling commit
                     for branch in repo.branches.with_commit(commit.hex):
                         if branch:
                             break
                     else:
                         # NOTE(marcink): Empty error doesn't give us any meaningful information
                         # here, we instead give something more explicit
                         e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
                         raise exceptions.LookupException(e)(missing_commit_err)
 
                 commit_id = commit.hex
                 type_id = commit.type
 
                 return {
                     'id': commit_id,
                     'type': self._type_id_to_name(type_id),
                     'commit_id': commit_id,
                     'idx': 0
                 }
 
         return _get_object(context_uid, repo_id, sha)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_refs(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 regex = re.compile('^refs/(heads|tags)/')
                 return {x.name: x.target.hex for x in
                         filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
 
         return _get_refs(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def get_branch_pointers(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_branch_pointers(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             regex = re.compile('^refs/heads')
             with repo_init as repo:
                 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
                 return {x.target.hex: x.shorthand for x in branches}
 
         return _get_branch_pointers(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def head(self, wire, show_exc=True):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _head(_context_uid, _repo_id, _show_exc):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     return repo.head.peel().hex
                 except Exception:
                     if show_exc:
                         raise
         return _head(context_uid, repo_id, show_exc)
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _revision(_context_uid, _repo_id, _rev):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[rev]
                 obj_data = {
                     'id': commit.id.hex,
                 }
                 # tree objects itself don't have tree_id attribute
                 if hasattr(commit, 'tree_id'):
                     obj_data['tree'] = commit.tree_id.hex
 
                 return obj_data
         return _revision(context_uid, repo_id, rev)
 
     @reraise_safe_exceptions
     def date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _date(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'commit_time'):
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
                 else:
                     commit = commit.get_object()
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
 
                 # TODO(marcink): check dulwich difference of offset vs timezone
                 return [commit_time, commit_time_offset]
         return _date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def author(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _author(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'author'):
                     author = commit.author
                 else:
                     author = commit.get_object().author
 
                 if author.email:
                     return u"{} <{}>".format(author.name, author.email)
 
                 try:
                     return u"{}".format(author.name)
                 except Exception:
                     return u"{}".format(safe_unicode(author.raw_name))
 
         return _author(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def message(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _message(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 return commit.message
         return _message(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _parents(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 if hasattr(commit, 'parent_ids'):
                     parent_ids = commit.parent_ids
                 else:
                     parent_ids = commit.get_object().parent_ids
 
                 return [x.hex for x in parent_ids]
         return _parents(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _children(_repo_id, _commit_id):
             output, __ = self.run_git_command(
                 wire, ['rev-list', '--all', '--children'])
 
             child_ids = []
             pat = re.compile(r'^%s' % commit_id)
             for l in output.splitlines():
                 if pat.match(l):
                     found_ids = l.split(' ')[1:]
                     child_ids.extend(found_ids)
 
             return child_ids
         return _children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.create(key, value, force=True)
 
     @reraise_safe_exceptions
     def create_branch(self, wire, branch_name, commit_id, force=False):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             commit = repo[commit_id]
 
             if force:
                 repo.branches.local.create(branch_name, commit, force=force)
             elif not repo.branches.get(branch_name):
                 # create only if that branch isn't existing
                 repo.branches.local.create(branch_name, commit, force=force)
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tag_remove(self, wire, tag_name):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             key = 'refs/tags/{}'.format(tag_name)
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
         # TODO(marcink): remove this seems it's only used by tests
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
     def tree_and_type_for_path(self, wire, commit_id, path):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
             repo_init = self._factory.repo_libgit2(wire)
 
             with repo_init as repo:
                 commit = repo[commit_id]
                 try:
                     tree = commit.tree[path]
                 except KeyError:
                     return None, None, None
 
                 return tree.id.hex, tree.type, tree.filemode
         return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_items(_repo_id, _tree_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     tree = repo[tree_id]
                 except KeyError:
                     raise ObjectMissing('No tree with id: {}'.format(tree_id))
 
                 result = []
                 for item in tree:
                     item_sha = item.hex
                     item_mode = item.filemode
                     item_type = item.type
 
                     if item_type == 'commit':
                         # NOTE(marcink): submodules we translate to 'link' for backward compat
                         item_type = 'link'
 
                     result.append((item.name, item_mode, item_sha, item_type))
                 return result
         return _tree_items(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         """
         Old version that uses subprocess to call diff
         """
 
         flags = [
             '-U%s' % context, '--patch',
             '--binary',
             '--find-renames',
             '--no-indent-heuristic',
             # '--indent-heuristic',
             #'--full-index',
             #'--abbrev=40'
         ]
 
         if opt_ignorews:
             flags.append('--ignore-all-space')
 
         if commit_id_1 == self.EMPTY_COMMIT:
             cmd = ['show'] + flags + [commit_id_2]
         else:
             cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
 
         if file_filter:
             cmd.extend(['--', file_filter])
 
         diff, __ = self.run_git_command(wire, cmd)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if commit_id_1 == self.EMPTY_COMMIT:
             lines = diff.splitlines()
             x = 0
             for line in lines:
                 if line.startswith('diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
             diff = '\n'.join(lines[x:]) + '\n'
         return diff
 
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             swap = True
             flags = 0
             flags |= pygit2.GIT_DIFF_SHOW_BINARY
 
             if opt_ignorews:
                 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
 
             if commit_id_1 == self.EMPTY_COMMIT:
                 comm1 = repo[commit_id_2]
                 diff_obj = comm1.tree.diff_to_tree(
                     flags=flags, context_lines=context, swap=swap)
 
             else:
                 comm1 = repo[commit_id_2]
                 comm2 = repo[commit_id_1]
                 diff_obj = comm1.tree.diff_to_tree(
                     comm2.tree, flags=flags, context_lines=context, swap=swap)
             similar_flags = 0
             similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
             diff_obj.find_similar(flags=similar_flags)
 
             if file_filter:
                 for p in diff_obj:
                     if p.delta.old_file.path == file_filter:
                         return p.patch or ''
                 # fo matching path == no diff
                 return ''
             return diff_obj.patch or ''
 
     @reraise_safe_exceptions
     def node_history(self, wire, commit_id, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
             # optimize for n==1, rev-list is much faster for that use-case
             if limit == 1:
                 cmd = ['rev-list', '-1', commit_id, '--', path]
             else:
                 cmd = ['log']
                 if limit:
                     cmd.extend(['-n', str(safe_int(limit, 0))])
                 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
 
             output, __ = self.run_git_command(wire, cmd)
             commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
 
             return [x for x in commit_ids]
         return _node_history(context_uid, repo_id, commit_id, path, limit)
 
     @reraise_safe_exceptions
     def node_annotate(self, wire, commit_id, path):
 
         cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for boundaries
         # -r commit_id ==> blames for the given commit
         output, __ = self.run_git_command(wire, cmd)
 
         result = []
         for i, blame_line in enumerate(output.split('\n')[:-1]):
             line_no = i + 1
             commit_id, line = re.split(r' ', blame_line, 1)
             result.append((line_no, commit_id, line))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         region = self._region(wire)
         @region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id):
 
             cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
             try:
                 output, __ = self.run_git_command(wire, cmd)
                 return output.splitlines()
             except Exception:
                 # Can be raised for empty repositories
                 return []
         return _get_all_commit_ids(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         if '_copts' in opts:
             _copts.extend(opts['_copts'] or [])
             del opts['_copts']
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
         _opts = {'env': gitenv, 'shell': False}
 
         proc = None
         try:
             _opts.update(opts)
             proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(proc), ''.join(proc.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
         finally:
             if proc:
                 proc.close()
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_git_hooks
         bare = self.bare(wire)
         path = wire['path']
         return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_git_pre_hook_version, get_git_post_hook_version)
         bare = self.bare(wire)
         path = wire['path']
         return {
             'pre_version': get_git_pre_hook_version(path, bare),
             'post_version': get_git_post_hook_version(path, bare),
         }
 
     @reraise_safe_exceptions
     def set_head_ref(self, wire, head_name):
         log.debug('Setting refs/head to `%s`', head_name)
-        cmd = ['symbolic-ref', 'HEAD', 'refs/heads/%s' % head_name]
+        cmd = ['symbolic-ref', '"HEAD"', '"refs/heads/%s"' % head_name]
         output, __ = self.run_git_command(wire, cmd)
         return [head_name] + output.splitlines()
 
     @reraise_safe_exceptions
     def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
                      archive_dir_name, commit_id):
 
         def file_walker(_commit_id, path):
             repo_init = self._factory.repo_libgit2(wire)
 
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if path in ['', '/']:
                     tree = commit.tree
                 else:
                     tree = commit.tree[path.rstrip('/')]
                     tree_id = tree.id.hex
                     try:
                         tree = repo[tree_id]
                     except KeyError:
                         raise ObjectMissing('No tree with id: {}'.format(tree_id))
 
                 index = LibGit2Index.Index()
                 index.read_tree(tree)
                 file_iter = index
 
                 for fn in file_iter:
                     file_path = fn.path
                     mode = fn.mode
                     is_link = stat.S_ISLNK(mode)
                     if mode == pygit2.GIT_FILEMODE_COMMIT:
                         log.debug('Skipping path %s as a commit node', file_path)
                         continue
                     yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw)
 
         return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
                             archive_dir_name, commit_id)

vcsserver/hooks.py

@@ -1,729 +1,729 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import logging
 import collections
 import importlib
 import base64
 
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import simplejson as json
 
 from vcsserver import exceptions, subprocessio, settings
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         try:
             connection.request('POST', '/', body)
         except Exception:
             log.error('Hooks calling Connection failed on %s', connection.__dict__)
             raise
         response = connection.getresponse()
 
         response_data = response.read()
 
         try:
             return json.loads(response_data)
         except Exception:
             log.exception('Failed to decode hook response json data. '
                           'response_code:%s, raw_data:%s',
                           response.status, response_data)
             raise
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class HooksShadowRepoClient(object):
 
     def __call__(self, hook_name, extras):
         return {'output': '', 'status': 0}
 
 
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 class SvnMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to svn clients."""
 
     def __init__(self, stderr=None):
         # SVN needs data sent to stderr for back-to-client messaging
         self.stderr = stderr or sys.stderr
 
     def write(self, message):
         self.stderr.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
     hooks_uri = extras.get('hooks_uri')
     is_shadow_repo = extras.get('is_shadow_repo')
     if hooks_uri:
         return HooksHttpClient(extras['hooks_uri'])
     elif is_shadow_repo:
         return HooksShadowRepoClient()
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks_client = _get_hooks_client(extras)
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
 
     _handle_exception(result)
     writer.write(result['output'])
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node, check_heads=False):
     from vcsserver.hgcompat import get_ctx
 
     commits = []
     revs = []
     start = get_ctx(repo, node).rev()
     end = len(repo)
     for rev in range(start, end):
         revs.append(rev)
         ctx = get_ctx(repo, rev)
         commit_id = mercurial.node.hex(ctx.node())
         branch = ctx.branch()
         commits.append((commit_id, branch))
 
     parent_heads = []
     if check_heads:
         parent_heads = _check_heads(repo, start, end, revs)
     return commits, parent_heads
 
 
 def _check_heads(repo, start, end, commits):
     from vcsserver.hgcompat import get_ctx
     changelog = repo.changelog
     parents = set()
 
     for new_rev in commits:
         for p in changelog.parentrevs(new_rev):
             if p == mercurial.node.nullrev:
                 continue
             if p < start:
                 parents.add(p)
 
     for p in parents:
         branch = get_ctx(repo, p).branch()
         # The heads descending from that parent, on the same branch
         parent_heads = set([p])
         reachable = set([p])
         for x in xrange(p + 1, end):
             if get_ctx(repo, x).branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
                 if pp in reachable:
                     reachable.add(x)
                     parent_heads.discard(pp)
                     parent_heads.add(x)
         # More than one head? Suggest merging
         if len(parent_heads) > 1:
             return list(parent_heads)
 
     return []
 
 
 def _get_git_env():
     env = {}
     for k, v in os.environ.items():
         if k.startswith('GIT'):
             env[k] = v
 
     # serialized version
     return [(k, v) for k, v in env.items()]
 
 
 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
     env = {}
     for k, v in os.environ.items():
         if k.startswith('HG'):
             env[k] = v
 
     env['HG_NODE'] = old_rev
     env['HG_NODE_LAST'] = new_rev
     env['HG_TXNID'] = txnid
     env['HG_PENDING'] = repo_path
 
     return [(k, v) for k, v in env.items()]
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook
     """
     extras = _extras_from_ui(ui)
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
     if node and kwargs.get('hooktype') == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
             old_rev = kwargs.get('node_last') or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
         for push_ref in rev_data:
             push_ref['multiple_heads'] = _heads
 
             repo_path = os.path.join(
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
                 new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
                 repo_path=repo_path)
 
     extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook for SSH
     """
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook
     """
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_branches'):
         bookmarks = ui._rc_pushkey_branches
 
     extras['hook_type'] = kwargs.get('hooktype', 'post_push')
     extras['commit_ids'] = commit_ids
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook for SSH
     """
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
     from vcsserver.hgcompat import get_ctx
     if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_branches = get_ctx(repo, kwargs['key']).bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 # NOTE(marcink):
                 # we're unable to tell total_commits for git at this point
                 # but we set the variable for consistency with GIT
                 'total_commits': -1,
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     empty_commit_id = '0' * 40
 
     detect_force_push = extras.get('detect_force_push')
 
     for push_ref in rev_data:
         # store our git-env which holds the temp store
         push_ref['git_env'] = _get_git_env()
         push_ref['pruned_sha'] = ''
         if not detect_force_push:
             # don't check for forced-push when we don't need to
             continue
 
         type_ = push_ref['type']
         new_branch = push_ref['old_rev'] == empty_commit_id
         delete_branch = push_ref['new_rev'] == empty_commit_id
         if type_ == 'heads' and not (new_branch or delete_branch):
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
             cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, '^{}'.format(new_rev)]
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push was used
             if stdout:
                 push_ref['pruned_sha'] = stdout.splitlines()
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
                 # starting new branch case
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 # Fix up head revision if needed
                 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
                 try:
                     subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
-                    cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
-                           'refs/heads/%s' % push_ref['name']]
+                    cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', '"HEAD"',
+                           '"refs/heads/%s"' % push_ref['name']]
                     print("Setting default branch to %s" % push_ref['name'])
                     subprocessio.run_command(cmd, env=os.environ.copy())
 
                 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
                        '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 heads = stdout
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
                 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
                        '--pretty=format:%H', '--', push_ref['new_rev'],
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE, 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['hook_type'] = 'post_receive'
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())
 
 
 def _get_extras_from_txn_id(path, txn_id):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-t', txn_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from txn_id')
 
     return extras
 
 
 def _get_extras_from_commit_id(commit_id, path):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-r', commit_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from commit_id')
 
     return extras
 
 
 def svn_pre_commit(repo_path, commit_data, env):
     path, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_txn_id(path, txn_id)
         if not extras:
             return 0
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = [txn_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'total_commits': 1,
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     return _call_hook('pre_push', extras, SvnMessageWriter())
 
 
 def svn_post_commit(repo_path, commit_data, env):
     """
     commit_data is path, rev, txn_id
     """
     if len(commit_data) == 3:
         path, commit_id, txn_id = commit_data
     elif len(commit_data) == 2:
         log.error('Failed to extract txn_id from commit_data using legacy method. '
                   'Some functionality might be limited')
         path, commit_id = commit_data
         txn_id = None
 
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_commit_id(commit_id, path)
         if not extras:
             return 0
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
         'total_commits': 1,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, SvnMessageWriter())
         except Exception:
             pass
 
     return _call_hook('post_push', extras, SvnMessageWriter())