release: Merge default into stable for release preparation

pkgs/patches/git/git-send-email-honor-PATH.patch

@@ -0,0 +1,26 b''
+diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt
+--- a/Documentation/git-send-email.txt
++++ b/Documentation/git-send-email.txt
+@@ -208,8 +208,7 @@ a password is obtained using 'git-credential'.
+ 	specify a full pathname of a sendmail-like program instead;
+ 	the program must support the `-i` option.  Default value can
+ 	be specified by the `sendemail.smtpServer` configuration
+-	option; the built-in default is to search for `sendmail` in
+-	`/usr/sbin`, `/usr/lib` and $PATH if such program is
++	option; the built-in default is to search in $PATH if such program is
+ 	available, falling back to `localhost` otherwise.
+
+ --smtp-server-port=<port>::
+diff --git a/git-send-email.perl b/git-send-email.perl
+--- a/git-send-email.perl
++++ b/git-send-email.perl
+@@ -944,8 +944,7 @@ if (defined $reply_to) {
+ }
+
+ if (!defined $smtp_server) {
+-	my @sendmail_paths = qw( /usr/sbin/sendmail /usr/lib/sendmail );
+-	push @sendmail_paths, map {"$_/sendmail"} split /:/, $ENV{PATH};
++	my @sendmail_paths = map {"$_/sendmail"} split /:/, $ENV{PATH};
+ 	foreach (@sendmail_paths) {
+ 		if (-x $_) {
+ 			$smtp_server = $_;

pkgs/patches/git/installCheck-path.patch

@@ -0,0 +1,12 b''
+diff --git a/t/test-lib.sh b/t/test-lib.sh
+--- a/t/test-lib.sh
++++ b/t/test-lib.sh
+@@ -923,7 +923,7 @@
+ then
+ 	GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path)  ||
+ 	error "Cannot run git from $GIT_TEST_INSTALLED."
+-	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR:$PATH
++	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$GIT_BUILD_DIR:$PATH
+ 	GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
+ else # normal case, use ../bin-wrappers only unless $with_dashes:
+ 	git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"

.bumpversion.cfg

@@ -1,6 +1,6 b''
 [bumpversion]
-current_version = 4.14.1
+current_version = 4.15.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]
 

.release.cfg

@@ -1,16 +1,14 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.14.1
+state = in_progress
+version = 4.15.0
 

pkgs/overlays.nix

@@ -1,45 +1,47 b''
 self: super: {
   # bump GIT version
   git = super.lib.overrideDerivation super.git (oldAttrs: {
-    name = "git-2.17.2";
+    name = "git-2.19.1";
     src = self.fetchurl {
-      url = "https://www.kernel.org/pub/software/scm/git/git-2.17.2.tar.xz";
-      sha256 = "1ghljlxmyqphx13qspy382cpl2pbkbwbhqm7w7z57r9mkhswx668";
+      url = "https://www.kernel.org/pub/software/scm/git/git-2.19.1.tar.xz";
+      sha256 = "1dfv43lmdnxz42504jc89sihbv1d4d6kgqcz3c5ji140kfm5cl1l";
     };
 
+    # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
     patches = [
       ./patches/git/docbook2texi.patch
-      ./patches/git/symlinks-in-bin.patch
       ./patches/git/git-sh-i18n.patch
       ./patches/git/ssh-path.patch
+      ./patches/git/git-send-email-honor-PATH.patch
+      ./patches/git/installCheck-path.patch
     ];
 
   });
 
   # Override subversion derivation to
   #  - activate python bindings
   subversion =
   let
     subversionWithPython = super.subversion.override {
       httpSupport = true;
       pythonBindings = true;
       python = self.python27Packages.python;
     };
   in
     super.lib.overrideDerivation subversionWithPython (oldAttrs: {
       name = "subversion-1.10.2";
       src = self.fetchurl {
         url = "https://archive.apache.org/dist/subversion/subversion-1.10.2.tar.gz";
         sha256 = "0xv5z2bg0lw7057g913yc13f60nfj257wvmsq22pr33m4syf26sg";
       };
 
       ## use internal lz4/utf8proc because it is stable and shipped with SVN
       configureFlags = oldAttrs.configureFlags ++ [
         " --with-lz4=internal"
         " --with-utf8proc=internal"
       ];
 
 
   });
 
 }

pkgs/patches/git/docbook2texi.patch

@@ -1,37 +1,37 b''
 This patch does two things: (1) use the right name for `docbook2texi',
 and (2) make sure `gitman.info' isn't produced since it's broken (duplicate
 node names).
 
-diff -ru git-1.8.4-orig/Documentation/Makefile git-1.8.4/Documentation/Makefile
---- git-1.8.4-orig/Documentation/Makefile	2013-08-23 21:38:43.000000000 +0200
-+++ git-1.8.4/Documentation/Makefile	2013-09-30 14:48:51.532890378 +0200
-@@ -101,7 +101,7 @@
- 
+diff --git a/Documentation/Makefile b/Documentation/Makefile
+--- a/Documentation/Makefile
++++ b/Documentation/Makefile
+@@ -122,7 +122,7 @@
+
  MAKEINFO = makeinfo
  INSTALL_INFO = install-info
 -DOCBOOK2X_TEXI = docbook2x-texi
 +DOCBOOK2X_TEXI = docbook2texi
  DBLATEX = dblatex
- ifndef PERL_PATH
- 	PERL_PATH = /usr/bin/perl
-@@ -205,7 +205,7 @@
+ ASCIIDOC_DBLATEX_DIR = /etc/asciidoc/dblatex
+ DBLATEX_COMMON = -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty
+@@ -240,7 +240,7 @@
  man5: $(DOC_MAN5)
  man7: $(DOC_MAN7)
- 
+
 -info: git.info gitman.info
 +info: git.info
- 
+
  pdf: user-manual.pdf
- 
-@@ -221,10 +221,9 @@
- 
+
+@@ -256,10 +256,9 @@
+
  install-info: info
  	$(INSTALL) -d -m 755 $(DESTDIR)$(infodir)
 -	$(INSTALL) -m 644 git.info gitman.info $(DESTDIR)$(infodir)
 +	$(INSTALL) -m 644 git.info $(DESTDIR)$(infodir)
  	if test -r $(DESTDIR)$(infodir)/dir; then \
  	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) git.info ;\
 -	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) gitman.info ;\
  	else \
  	  echo "No directory found in $(DESTDIR)$(infodir)" >&2 ; \
  	fi

pkgs/python-packages.nix

@@ -1,950 +1,950 b''
 # Generated by pip2nix 0.8.0.dev1
 # See https://github.com/johbo/pip2nix
 
 { pkgs, fetchurl, fetchgit, fetchhg }:
 
 self: super: {
   "atomicwrites" = super.buildPythonPackage {
     name = "atomicwrites-1.2.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/ed/a311712ef6b4355035489f665e63e1a73f9eb371929e3c98e5efd451069e/atomicwrites-1.2.1.tar.gz";
       sha256 = "1vmkbw9j0qammwxbxycrs39gvdg4lc2d4lk98kwf8ag2manyi6pc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "attrs" = super.buildPythonPackage {
     name = "attrs-18.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0f/9e/26b1d194aab960063b266170e53c39f73ea0d0d3f5ce23313e0ec8ee9bdf/attrs-18.2.0.tar.gz";
       sha256 = "0s9ydh058wmmf5v391pym877x4ahxg45dw6a0w4c7s5wgpigdjqh";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "backports.shutil-get-terminal-size" = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "beautifulsoup4" = super.buildPythonPackage {
     name = "beautifulsoup4-4.6.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz";
       sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "configobj" = super.buildPythonPackage {
     name = "configobj-5.0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c";
       sha256 = "1hhcxirwvg58grlfr177b3awhbq8hlx1l3lh69ifl1ki7lfd1s1x";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "coverage" = super.buildPythonPackage {
     name = "coverage-4.5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/35/fe/e7df7289d717426093c68d156e0fd9117c8f4872b6588e8a8928a0f68424/coverage-4.5.1.tar.gz";
       sha256 = "1wbrzpxka3xd4nmmkc6q0ir343d91kymwsm8pbmwa0d2a7q4ir2n";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "decorator" = super.buildPythonPackage {
     name = "decorator-4.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz";
       sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   "dogpile.cache" = super.buildPythonPackage {
     name = "dogpile.cache-0.6.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ee/bd/440da735a11c6087eed7cc8747fc4b995cbac2464168682f8ee1c8e43844/dogpile.cache-0.6.7.tar.gz";
       sha256 = "1aw8rx8vhb75y7zc6gi67g21sw057jdx7i8m3jq7kf3nqavxx9zw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dogpile.core" = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dulwich" = super.buildPythonPackage {
     name = "dulwich-0.13.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz";
       sha256 = "0f1jwvrh549c4rgavkn3wizrch904s73s4fmrxykxy9cw8s57lwf";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "enum34" = super.buildPythonPackage {
     name = "enum34-1.1.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
       sha256 = "1cgm5ng2gcfrkrm3hc22brl6chdmv67b9zvva9sfs7gn7dwc9n4a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "funcsigs" = super.buildPythonPackage {
     name = "funcsigs-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz";
       sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7";
     };
     meta = {
       license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "gevent" = super.buildPythonPackage {
-    name = "gevent-1.3.6";
+    name = "gevent-1.3.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."greenlet"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/49/13/aa4bb3640b5167fe58875d3d7e65390cdb14f9682a41a741a566bb560842/gevent-1.3.6.tar.gz";
-      sha256 = "1ih4k73dqz2zb561hda99vbanja3m6cdch3mgxxn1mla3qwkqhbv";
+      url = "https://files.pythonhosted.org/packages/10/c1/9499b146bfa43aa4f1e0ed1bab1bd3209a4861d25650c11725036c731cf5/gevent-1.3.7.tar.gz";
+      sha256 = "0b0fr04qdk1p4sniv87fh8z5psac60x01pv054kpgi94520g81iz";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gprof2dot" = super.buildPythonPackage {
     name = "gprof2dot-2017.9.19";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz";
       sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f";
     };
     meta = {
       license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ];
     };
   };
   "greenlet" = super.buildPythonPackage {
     name = "greenlet-0.4.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz";
       sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gunicorn" = super.buildPythonPackage {
     name = "gunicorn-19.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz";
       sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "hg-evolve" = super.buildPythonPackage {
     name = "hg-evolve-8.0.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/06/1a/c5c12d8f117426f05285a820ee5a23121882f5381104e86276b72598934f/hg-evolve-8.0.1.tar.gz";
       sha256 = "1brafifb42k71gl7qssb5m3ijnm7y30lfvm90z8xxcr2fgz19p29";
     };
     meta = {
       license = [ { fullName = "GPLv2+"; } ];
     };
   };
   "hgsubversion" = super.buildPythonPackage {
-    name = "hgsubversion-1.9.2";
+    name = "hgsubversion-1.9.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."mercurial"
       self."subvertpy"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/05/80/3a3cef10dd65e86528ef8d7ac57a41ebc782d0f3c6cfa4fed021aa9fbee0/hgsubversion-1.9.2.tar.gz";
-      sha256 = "16490narhq14vskml3dam8g5y3w3hdqj3g8bgm2b0c0i85l1xvcz";
+      url = "https://files.pythonhosted.org/packages/a3/53/6d205e641f3e09abcf1ddaed66e5e4b20da22d0145566d440a02c9e35f0d/hgsubversion-1.9.3.tar.gz";
+      sha256 = "0nymcjlch8c4zjbncrs30p2nrbylsf25g3h6mr0zzzxr141h3sig";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   "hupper" = super.buildPythonPackage {
-    name = "hupper-1.3.1";
+    name = "hupper-1.4.2";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/cf/4b/467b826a84c8594b81f414b5ab6794e981951dac90ca40abaf9ea1cb36b0/hupper-1.3.1.tar.gz";
-      sha256 = "03mf13n6i4dd60wlb9m99ddl4m3lmly70cjp7f82vdkibfl1v6l9";
+      url = "https://files.pythonhosted.org/packages/f1/75/1915dc7650b4867fa3049256e24ca8eddb5989998fcec788cf52b9812dfc/hupper-1.4.2.tar.gz";
+      sha256 = "16vb9fkiaakdpcp6pn56h3w0dwvm67bxq2k2dv4i382qhqwphdzb";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "ipdb" = super.buildPythonPackage {
     name = "ipdb-0.11";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."ipython"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/80/fe/4564de08f174f3846364b3add8426d14cebee228f741c27e702b2877e85b/ipdb-0.11.tar.gz";
       sha256 = "02m0l8wrhhd3z7dg3czn5ys1g5pxib516hpshdzp7rxzsxgcd0bh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython" = super.buildPythonPackage {
     name = "ipython-5.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."decorator"
       self."pickleshare"
       self."simplegeneric"
       self."traitlets"
       self."prompt-toolkit"
       self."pygments"
       self."pexpect"
       self."backports.shutil-get-terminal-size"
       self."pathlib2"
       self."pexpect"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython-genutils" = super.buildPythonPackage {
     name = "ipython-genutils-0.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz";
       sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mako" = super.buildPythonPackage {
     name = "mako-1.0.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/eb/f3/67579bb486517c0d49547f9697e36582cd19dafb5df9e687ed8e22de57fa/Mako-1.0.7.tar.gz";
       sha256 = "1bi5gnr8r8dva06qpyx4kgjc6spm2k1y908183nbbaylggjzs0jf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "markupsafe" = super.buildPythonPackage {
     name = "markupsafe-1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz";
       sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mercurial" = super.buildPythonPackage {
     name = "mercurial-4.6.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d9/fb/c7ecf2b7fd349878dbf45b8390b8db735cef73d49dd9ce8a364b4ca3a846/mercurial-4.6.2.tar.gz";
       sha256 = "1bv6wgcdx8glihjjfg22khhc52mclsn4kwfqvzbzlg0b42h4xl0w";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "mock" = super.buildPythonPackage {
     name = "mock-1.0.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz";
       sha256 = "0kzlsbki6q0awf89rc287f3aj8x431lrajf160a70z0ikhnxsfdq";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "more-itertools" = super.buildPythonPackage {
     name = "more-itertools-4.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/ff/6d485d7362f39880810278bdc906c13300db05485d9c65971dec1142da6a/more-itertools-4.3.0.tar.gz";
       sha256 = "17h3na0rdh8xq30w4b9pizgkdxmm51896bxw600x84jflg9vaxn4";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "msgpack-python" = super.buildPythonPackage {
     name = "msgpack-python-0.5.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz";
       sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "pastedeploy" = super.buildPythonPackage {
     name = "pastedeploy-1.5.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz";
       sha256 = "1jz3m4hq8v6hyhfjz9425nd3nvn52cvbfipdcd72krjmla4qz1fm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pathlib2" = super.buildPythonPackage {
     name = "pathlib2-2.3.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."scandir"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/db/a8/7d6439c1aec525ed70810abee5b7d7f3aa35347f59bc28343e8f62019aa2/pathlib2-2.3.2.tar.gz";
       sha256 = "10yb0iv5x2hs631rcppkhbddx799d3h8pcwmkbh2a66ns3w71ccf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pexpect" = super.buildPythonPackage {
     name = "pexpect-4.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ptyprocess"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/43/07d07654ee3e25235d8cea4164cdee0ec39d1fda8e9203156ebe403ffda4/pexpect-4.6.0.tar.gz";
       sha256 = "1fla85g47iaxxpjhp9vkxdnv4pgc7rplfy6ja491smrrk0jqi3ia";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   "pickleshare" = super.buildPythonPackage {
     name = "pickleshare-0.7.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz";
       sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster" = super.buildPythonPackage {
     name = "plaster-1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz";
       sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster-pastedeploy" = super.buildPythonPackage {
     name = "plaster-pastedeploy-0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."pastedeploy"
       self."plaster"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3f/e7/6a6833158d2038ec40085433308a1e164fd1dac595513f6dd556d5669bb8/plaster_pastedeploy-0.6.tar.gz";
       sha256 = "1bkggk18f4z2bmsmxyxabvf62znvjwbivzh880419r3ap0616cf2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pluggy" = super.buildPythonPackage {
     name = "pluggy-0.8.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/65/25/81d0de17cd00f8ca994a4e74e3c4baf7cd25072c0b831dad5c7d9d6138f8/pluggy-0.8.0.tar.gz";
       sha256 = "1580p47l2zqzsza8jcnw1h2wh3vvmygk6ly8bvi4w0g8j14sjys4";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "prompt-toolkit" = super.buildPythonPackage {
     name = "prompt-toolkit-1.0.15";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."wcwidth"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz";
       sha256 = "05v9h5nydljwpj5nm8n804ms0glajwfy1zagrzqrg91wk3qqi1c5";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psutil" = super.buildPythonPackage {
     name = "psutil-5.4.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/9a/1e93d41708f8ed2b564395edfa3389f0fd6d567597401c2e5e2775118d8b/psutil-5.4.7.tar.gz";
       sha256 = "0fsgmvzwbdbszkwfnqhib8jcxm4w6zyhvlxlcda0rfm5cyqj4qsv";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ptyprocess" = super.buildPythonPackage {
     name = "ptyprocess-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz";
       sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj";
     };
     meta = {
       license = [  ];
     };
   };
   "py" = super.buildPythonPackage {
     name = "py-1.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4f/38/5f427d1eedae73063ce4da680d2bae72014995f9fdeaa57809df61c968cd/py-1.6.0.tar.gz";
       sha256 = "1wcs3zv9wl5m5x7p16avqj2gsrviyb23yvc3pr330isqs0sh98q6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pygments" = super.buildPythonPackage {
-    name = "pygments-2.2.0";
+    name = "pygments-2.3.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz";
-      sha256 = "1k78qdvir1yb1c634nkv6rbga8wv4289xarghmsbbvzhvr311bnv";
+      url = "https://files.pythonhosted.org/packages/63/a2/91c31c4831853dedca2a08a0f94d788fc26a48f7281c99a303769ad2721b/Pygments-2.3.0.tar.gz";
+      sha256 = "1z34ms51dh4jq4h3cizp7vd1dmsxcbvffkjsd2xxfav22nn6lrl2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyramid" = super.buildPythonPackage {
     name = "pyramid-1.9.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."webob"
       self."repoze.lru"
       self."zope.interface"
       self."zope.deprecation"
       self."venusian"
       self."translationstring"
       self."pastedeploy"
       self."plaster"
       self."plaster-pastedeploy"
       self."hupper"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a0/c1/b321d07cfc4870541989ad131c86a1d593bfe802af0eca9718a0dadfb97a/pyramid-1.9.2.tar.gz";
       sha256 = "09drsl0346nchgxp2j7sa5hlk7mkhfld9wvbd0wicacrp26a92fg";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-mako" = super.buildPythonPackage {
     name = "pyramid-mako-1.0.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."mako"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
       sha256 = "18gk2vliq8z4acblsl6yzgbvnr9rlxjlcqir47km7kvlk1xri83d";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pytest" = super.buildPythonPackage {
     name = "pytest-3.8.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."py"
       self."six"
       self."setuptools"
       self."attrs"
       self."more-itertools"
       self."atomicwrites"
       self."pluggy"
       self."funcsigs"
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5f/d2/7f77f406ac505abda02ab4afb50d06ebf304f6ea42fca34f8f37529106b2/pytest-3.8.2.tar.gz";
       sha256 = "18nrwzn61kph2y6gxwfz9ms68rfvr9d4vcffsxng9p7jk9z18clk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-cov" = super.buildPythonPackage {
     name = "pytest-cov-2.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d9/e2/58f90a316fbd94dd50bf5c826a23f3f5d079fb3cc448c1e9f0e3c33a3d2a/pytest-cov-2.6.0.tar.gz";
       sha256 = "0qnpp9y3ygx4jk4pf5ad71fh2skbvnr6gl54m7rg5qysnx4g0q73";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   "pytest-profiling" = super.buildPythonPackage {
     name = "pytest-profiling-1.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."pytest"
       self."gprof2dot"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f5/34/4626126e041a51ef50a80d0619519b18d20aef249aac25b0d0fdd47e57ee/pytest-profiling-1.3.0.tar.gz";
       sha256 = "08r5afx5z22yvpmsnl91l4amsy1yxn8qsmm61mhp06mz8zjs51kb";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-runner" = super.buildPythonPackage {
     name = "pytest-runner-4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9e/b7/fe6e8f87f9a756fd06722216f1b6698ccba4d269eac6329d9f0c441d0f93/pytest-runner-4.2.tar.gz";
       sha256 = "1gkpyphawxz38ni1gdq1fmwyqcg02m7ypzqvv46z06crwdxi2gyj";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-sugar" = super.buildPythonPackage {
     name = "pytest-sugar-0.9.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."termcolor"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3e/6a/a3f909083079d03bde11d06ab23088886bbe25f2c97fbe4bb865e2bf05bc/pytest-sugar-0.9.1.tar.gz";
       sha256 = "0b4av40dv30727m54v211r0nzwjp2ajkjgxix6j484qjmwpw935b";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pytest-timeout" = super.buildPythonPackage {
     name = "pytest-timeout-1.3.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8c/3e/1b6a319d12ae7baa3acb7c18ff2c8630a09471a0319d43535c683b4d03eb/pytest-timeout-1.3.2.tar.gz";
       sha256 = "09wnmzvnls2mnsdz7x3c3sk2zdp6jl4dryvyj5i8hqz16q2zq5qi";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   "repoze.lru" = super.buildPythonPackage {
     name = "repoze.lru-0.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz";
       sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "rhodecode-vcsserver" = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.14.1";
+    name = "rhodecode-vcsserver-4.15.0";
     buildInputs = [
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."webtest"
       self."cov-core"
       self."coverage"
       self."configobj"
     ];
     doCheck = true;
     propagatedBuildInputs = [
       self."configobj"
       self."atomicwrites"
       self."attrs"
       self."dogpile.cache"
       self."dogpile.core"
       self."decorator"
       self."dulwich"
       self."hgsubversion"
       self."hg-evolve"
       self."mako"
       self."markupsafe"
       self."mercurial"
       self."msgpack-python"
       self."pastedeploy"
       self."psutil"
       self."pyramid"
       self."pyramid-mako"
       self."pygments"
       self."pathlib2"
       self."repoze.lru"
       self."simplejson"
       self."subprocess32"
       self."subvertpy"
       self."six"
       self."translationstring"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."gevent"
       self."greenlet"
       self."gunicorn"
       self."waitress"
       self."setproctitle"
       self."ipdb"
       self."ipython"
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."webtest"
       self."cov-core"
       self."coverage"
     ];
     src = ./.;
     meta = {
       license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
     };
   };
   "scandir" = super.buildPythonPackage {
     name = "scandir-1.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz";
       sha256 = "0r3hvf1a9jm1rkqgx40gxkmccknkaiqjavs8lccgq9s8khh5x5s4";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
     };
   };
   "setproctitle" = super.buildPythonPackage {
     name = "setproctitle-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz";
       sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "setuptools" = super.buildPythonPackage {
-    name = "setuptools-40.4.3";
+    name = "setuptools-40.6.2";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/6e/9c/6a003320b00ef237f94aa74e4ad66c57a7618f6c79d67527136e2544b728/setuptools-40.4.3.zip";
-      sha256 = "058v6zns4634n4al2nmmvp15j8nrgwn8wjrbdks47wk3vm05gg5c";
+      url = "https://files.pythonhosted.org/packages/b0/d1/8acb42f391cba52e35b131e442e80deffbb8d0676b93261d761b1f0ef8fb/setuptools-40.6.2.zip";
+      sha256 = "0r2c5hapirlzm34h7pl1lgkm6gk7bcrlrdj28qgsvaqg3f74vfw6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "simplejson" = super.buildPythonPackage {
     name = "simplejson-3.11.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/08/48/c97b668d6da7d7bebe7ea1817a6f76394b0ec959cb04214ca833c34359df/simplejson-3.11.1.tar.gz";
       sha256 = "1rr58dppsq73p0qcd9bsw066cdd3v63sqv7j6sqni8frvm4jv8h1";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   "six" = super.buildPythonPackage {
     name = "six-1.11.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
       sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "subprocess32" = super.buildPythonPackage {
     name = "subprocess32-3.5.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c3/5f/7117737fc7114061837a4f51670d863dd7f7f9c762a6546fa8a0dcfe61c8/subprocess32-3.5.2.tar.gz";
       sha256 = "11v62shwmdys48g7ncs3a8jwwnkcl8d4zcwy6dk73z1zy2f9hazb";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "subvertpy" = super.buildPythonPackage {
     name = "subvertpy-0.10.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/76/99fa82affce75f5ac0f7dbe513796c3f37311ace0c68e1b063683b4f9b99/subvertpy-0.10.1.tar.gz";
       sha256 = "061ncy9wjz3zyv527avcrdyk0xygyssyy7p1644nhzhwp8zpybij";
     };
     meta = {
       license = [ pkgs.lib.licenses.lgpl21Plus pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "termcolor" = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "traitlets" = super.buildPythonPackage {
     name = "traitlets-4.3.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."six"
       self."decorator"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a5/98/7f5ef2fe9e9e071813aaf9cb91d1a732e0a68b6c44a32b38cb8e14c3f069/traitlets-4.3.2.tar.gz";
       sha256 = "0dbq7sx26xqz5ixs711k5nc88p8a0nqyz6162pwks5dpcz9d4jww";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "translationstring" = super.buildPythonPackage {
     name = "translationstring-1.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "venusian" = super.buildPythonPackage {
     name = "venusian-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/38/24/b4b470ab9e0a2e2e9b9030c7735828c8934b4c6b45befd1bb713ec2aeb2d/venusian-1.1.0.tar.gz";
       sha256 = "0zapz131686qm0gazwy8bh11vr57pr89jbwbl50s528sqy9f80lr";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "waitress" = super.buildPythonPackage {
     name = "waitress-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3c/68/1c10dd5c556872ceebe88483b0436140048d39de83a84a06a8baa8136f4f/waitress-1.1.0.tar.gz";
       sha256 = "1a85gyji0kajc3p0s1pwwfm06w4wfxjkvvl4rnrz3h164kbd6g6k";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "wcwidth" = super.buildPythonPackage {
     name = "wcwidth-0.1.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
       sha256 = "0pn6dflzm609m4r3i8ik5ni9ijjbb5fa3vg1n7hn6vkd49r77wrx";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webob" = super.buildPythonPackage {
     name = "webob-1.7.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/75/34/731e23f52371852dfe7490a61644826ba7fe70fd52a377aaca0f4956ba7f/WebOb-1.7.4.tar.gz";
       sha256 = "1na01ljg04z40il7vcrn8g29vaw7nvg1xvhk64cr4jys5wcay44d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webtest" = super.buildPythonPackage {
     name = "webtest-2.0.29";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webob"
       self."waitress"
       self."beautifulsoup4"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/de/8f94738be649997da99c47b104aa3c3984ecec51a1d8153ed09638253d56/WebTest-2.0.29.tar.gz";
       sha256 = "0bcj1ica5lnmj5zbvk46x28kgphcsgh7sfnwjmn0cr94mhawrg6v";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "zope.deprecation" = super.buildPythonPackage {
     name = "zope.deprecation-4.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz";
       sha256 = "095jas41wbxgmw95kwdxqhbc3bgihw2hzj9b3qpdg85apcsf2lkx";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.interface" = super.buildPythonPackage {
     name = "zope.interface-4.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/8a/657532df378c2cd2a1fe6b12be3b4097521570769d4852ec02c24bd3594e/zope.interface-4.5.0.tar.gz";
       sha256 = "0k67m60ij06wkg82n15qgyn96waf4pmrkhv0njpkfzpmv5q89hsp";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
 
 ### Test requirements
 
   
 }

requirements.txt

@@ -1,48 +1,48 b''
 ## dependencies
 
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c#egg=configobj==5.0.6
 atomicwrites==1.2.1
 attrs==18.2.0
 dogpile.cache==0.6.7
 dogpile.core==0.4.1
 decorator==4.1.2
 dulwich==0.13.0
-hgsubversion==1.9.2
+hgsubversion==1.9.3
 hg-evolve==8.0.1
 mako==1.0.7
 markupsafe==1.0.0
 mercurial==4.6.2
 msgpack-python==0.5.6
 
 pastedeploy==1.5.2
 psutil==5.4.7
 pyramid==1.9.2
 pyramid-mako==1.0.2
 
-pygments==2.2.0
+pygments==2.3.0
 pathlib2==2.3.2
 repoze.lru==0.7
 simplejson==3.11.1
 subprocess32==3.5.2
 subvertpy==0.10.1
 
 six==1.11.0
 translationstring==1.3
 webob==1.7.4
 zope.deprecation==4.3.0
 zope.interface==4.5.0
 
 ## http servers
-gevent==1.3.6
+gevent==1.3.7
 greenlet==0.4.15
 gunicorn==19.9.0
 waitress==1.1.0
 setproctitle==1.1.10
 
 ## debug
 ipdb==0.11.0
 ipython==5.1.0
 
 ## test related requirements
 -r requirements_test.txt

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.14.1
  No newline at end of file
+4.15.0
  No newline at end of file

vcsserver/git.py

@@ -1,728 +1,719 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 import collections
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import traceback
 import urllib
 import urllib2
 from functools import wraps
 
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo, Tag
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str
 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 
 log = logging.getLogger(__name__)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError,
                 ObjectMissing) as e:
             exc = exceptions.LookupException(e)
             raise exc(e)
         except (HangupException, UnexpectedCommandError) as e:
             exc = exceptions.VcsException(e)
             raise exc(e)
         except Exception as e:
             # NOTE(marcink): becuase of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
                 #log.exception("Unhandled exception in git remote call")
                 #raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
 
     TODO: mikhail: please check if we need this wrapper after updating dulwich
     to 0.12.0 +
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class GitFactory(RepoFactory):
     repo_type = 'git'
 
     def _create_repo(self, wire, create):
         repo_path = str_to_dulwich(wire['path'])
         return Repo(repo_path)
 
 
 class GitRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
         self.peeled_ref_marker = '^{}'
         self._bulk_methods = {
             "author": self.commit_attribute,
             "date": self.get_object_attrs,
             "message": self.commit_attribute,
             "parents": self.commit_attribute,
             "_commit": self.revision,
         }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
             return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
         return {}
 
     def _assign_ref(self, wire, ref, commit_id):
         repo = self._factory.repo(wire)
         repo[ref] = commit_id
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo = self._factory.repo(wire)
         blob = objects.Blob()
         blob.set_raw_string(content)
         repo.object_store.add_object(blob)
         return blob.id
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         path = wire.get('path')
         try:
             self._factory.repo(wire)
         except NotGitRepository as e:
             tb = traceback.format_exc()
             log.debug("Invalid Git path `%s`, tb: %s", path, tb)
             return False
 
         return True
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo = self._factory.repo(wire)
         return repo.bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo = self._factory.repo(wire)
         return repo[sha].as_pretty_string()
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         repo = self._factory.repo(wire)
         blob = repo[sha]
         return blob.raw_length()
 
     def _parse_lfs_pointer(self, raw_content):
 
         spec_string = 'version https://git-lfs.github.com/spec'
         if raw_content and raw_content.startswith(spec_string):
             pattern = re.compile(r"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
             ^size[ ](?P<oid_size>[0-9]+)\n
             (?:\n)?
             """, re.VERBOSE | re.MULTILINE)
             match = pattern.match(raw_content)
             if match:
                 return match.groupdict()
 
         return {}
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, sha):
         repo = self._factory.repo(wire)
         blob = repo[sha]
         return self._parse_lfs_pointer(blob.as_raw_string())
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
         repo = self._factory.repo(wire)
         conf = self._wire_to_config(wire)
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
             repo_name = repo.path
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.has_oid()
 
         return False
 
     @reraise_safe_exceptions
     def store_path(self, wire, oid):
         repo = self._factory.repo(wire)
         conf = self._wire_to_config(wire)
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
             repo_name = repo.path
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
         raise ValueError('Unable to fetch oid with path {}'.format(oid))
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 args = [wire, rev]
                 if attr == "date":
                     args.extend(["commit_time", "commit_timezone"])
                 elif attr in ["author", "message", "parents"]:
                     args.append(attr)
                 result[attr] = method(*args)
             except KeyError as e:
                 raise exceptions.VcsException(e)(
                     "Unknown bulk attribute: %s" % attr)
         return result
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError()(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
         remote_refs = self.pull(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
         commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
                 parent.add(
                     name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.iteritems():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             for k in remote_refs:
                 if k.endswith(self.peeled_ref_marker):
                     log.debug("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
                 repo['HEAD'] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_fetch(self, wire, url, refs=None):
         repo = self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
 
         # get all remote refs we'll use to fetch later
         output, __ = self.run_git_command(
             wire, ['ls-remote', url], fail_on_stderr=False,
             _copts=['-c', 'core.askpass=""'],
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         remote_refs = collections.OrderedDict()
         fetch_refs = []
 
         for ref_line in output.splitlines():
             sha, ref = ref_line.split('\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
                 continue
             if ref.endswith(self.peeled_ref_marker):
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
             if ref in ['HEAD']:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
                 fetch_refs.append('{}:{}'.format(ref, ref))
             elif not refs:
                 fetch_refs.append('{}:{}'.format(ref, ref))
 
         if fetch_refs:
             _out, _err = self.run_git_command(
                 wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs,
                 fail_on_stderr=False,
                 _copts=['-c', 'core.askpass=""'],
                 extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url, refs=None):
         if not self.check_url(url, wire):
             return
 
         repo = self._factory.repo(wire)
         self.run_git_command(
             wire, ['push', url, '--mirror'], fail_on_stderr=False,
             _copts=['-c', 'core.askpass=""'],
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
-    def get_file_history(self, wire, file_path, commit_id, limit):
-        repo = self._factory.repo(wire)
-        include = [commit_id]
-        paths = [file_path]
-
-        walker = repo.get_walker(include, paths=paths, max_entries=limit)
-        return [x.commit.id for x in walker]
-
-    @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         commit_id = obj.id
 
         if isinstance(obj, Tag):
             commit_id = obj.object[1]
 
         return {
             'id': obj.id,
             'type': obj.type_name,
             'commit_id': commit_id
         }
 
     @reraise_safe_exceptions
     def get_object_attrs(self, wire, sha, *attrs):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         return list(getattr(obj, a) for a in attrs)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         repo = self._factory.repo(wire)
         result = {}
         for ref, sha in repo.refs.as_dict().items():
             peeled_sha = repo.get_peeled(ref)
             result[ref] = peeled_sha
         return result
 
     @reraise_safe_exceptions
     def get_refs_path(self, wire):
         repo = self._factory.repo(wire)
         return repo.refs.path
 
     @reraise_safe_exceptions
     def head(self, wire, show_exc=True):
         repo = self._factory.repo(wire)
         try:
             return repo.head()
         except Exception:
             if show_exc:
                 raise
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         obj_data = {
             'id': obj.id,
         }
         try:
             obj_data['tree'] = obj.tree
         except AttributeError:
             pass
         return obj_data
 
     @reraise_safe_exceptions
     def commit_attribute(self, wire, rev, attr):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         return getattr(obj, attr)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo = self._factory.repo(wire)
         repo.refs[key] = value
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo = self._factory.repo(wire)
         del repo.refs[key]
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         repo = self._factory.repo(wire)
         tree = repo[tree_id]
 
         result = []
         for item in tree.iteritems():
             item_sha = item.sha
             item_mode = item.mode
 
             if FILE_MODE(item_mode) == GIT_LINK:
                 item_type = "link"
             else:
                 item_type = repo[item_sha].type_name
 
             result.append((item.path, item_mode, item_sha, item_type))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = 'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return stdout.strip()
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         if '_copts' in opts:
             _copts.extend(opts['_copts'] or [])
             del opts['_copts']
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
         _opts = {'env': gitenv, 'shell': False}
 
         try:
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(p), ''.join(p.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_git_hooks
         repo = self._factory.repo(wire)
         return install_git_hooks(repo.path, repo.bare, force_create=force)
 
 
 def str_to_dulwich(value):
     """
     Dulwich 0.10.1a requires `unicode` objects to be passed in.
     """
     return value.decode(settings.WIRE_ENCODING)

vcsserver/hg.py

@@ -1,795 +1,795 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import logging
 import stat
 import urllib
 import urllib2
 
 from hgext import largefiles, rebase
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
 from mercurial import verify
 
 from vcsserver import exceptions
 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     makepeer, localrepository, match, memctx, exchange, memfilectx, nullrev,
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError)
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     baseui.setconfig('ui', 'paginate', 'never')
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig('worker', 'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig('extensions', 'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig('extensions', 'largefiles', '!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired) as e:
             raise_from_original(exceptions.AbortException(e))
         except RepoLookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except RequirementError as e:
             raise_from_original(exceptions.RequirementException(e))
         except RepoError as e:
             raise_from_original(exceptions.VcsException(e))
         except LookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException(e))
 
             raise
     return wrapper
 
 
 class MercurialFactory(RepoFactory):
     repo_type = 'hg'
 
     def _create_config(self, config, hooks=True):
         if not hooks:
             hooks_to_clean = frozenset((
                 'changegroup.repo_size', 'preoutgoing.pre_pull',
                 'outgoing.pull_logger', 'prechangegroup.pre_push'))
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         return localrepository(baseui, wire["path"], create)
 
 
 class HgRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
 
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "obsolete": self.ctx_obsolete,
             "phase": self.ctx_phase,
             "hidden": self.ctx_hidden,
             "_file_paths": self.ctx_list,
         }
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return util.version()
 
     @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
         elif kind == "tbz2":
             archiver = archival.tarit(archive_path, mtime, "bz2")
         elif kind == 'zip':
             archiver = archival.zipit(archive_path, mtime)
         else:
             raise exceptions.ArchiveException()(
                 'Remote does not support: "%s".' % kind)
 
         for f_path, f_mode, f_is_link, f_content in file_info:
             archiver.addfile(f_path, f_mode, f_is_link, f_content)
         archiver.done()
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
         repo = self._factory.repo(wire)
         return dict(repo._bookmarks)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
         repo = self._factory.repo(wire)
         iter_branches = repo.branchmap().iterbranches()
         bt = {}
         for branch_name, _heads, tip, is_closed in iter_branches:
             if normal and not is_closed:
                 bt[branch_name] = tip
             if closed and is_closed:
                 bt[branch_name] = tip
 
         return bt
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 result[attr] = method(wire, rev)
             except KeyError as e:
                 raise exceptions.VcsException(e)(
                     'Unknown bulk attribute: "%s"' % attr)
         return result
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
         clone(baseui, source, dest, noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(
             self, wire, message, parents, commit_time, commit_timezone,
             user, files, extra, removed, updated):
 
         def _filectxfn(_repo, memctx, path):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if path in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
                 if node['path'] == path:
                     return memfilectx(
                         _repo,
                         changectx=memctx,
                         path=node['path'],
                         data=node['content'],
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copied=False)
 
             raise exceptions.AbortException()(
                 "Given path haven't been marked as added, "
                 "changed or removed (%s)" % path)
 
         repo = self._factory.repo(wire)
 
         commit_ctx = memctx(
             repo=repo,
             parents=parents,
             text=message,
             files=files,
             filectxfn=_filectxfn,
             user=user,
             date=(commit_time, commit_timezone),
             extra=extra)
 
         n = repo.commitctx(commit_ctx)
         new_id = hex(n)
 
         return new_id
 
     @reraise_safe_exceptions
     def ctx_branch(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.branch()
 
     @reraise_safe_exceptions
     def ctx_children(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [child.rev() for child in ctx.children()]
 
     @reraise_safe_exceptions
     def ctx_date(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.date()
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.description()
 
     @reraise_safe_exceptions
     def ctx_diff(
             self, wire, revision, git=True, ignore_whitespace=True, context=3):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         result = ctx.diff(
             git=git, ignore_whitespace=ignore_whitespace, context=context)
         return list(result)
 
     @reraise_safe_exceptions
     def ctx_files(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.files()
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = repo[revision]
         return list(ctx)
 
     @reraise_safe_exceptions
     def ctx_parents(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [parent.rev() for parent in ctx.parents()]
 
     @reraise_safe_exceptions
     def ctx_phase(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         # public=0, draft=1, secret=3
         return ctx.phase()
 
     @reraise_safe_exceptions
     def ctx_obsolete(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.obsolete()
 
     @reraise_safe_exceptions
     def ctx_hidden(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.hidden()
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
 
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
                     "Verifying if URL is a Mercurial repository: %s",
                     cleaned_uri)
                 ui = make_ui_from_config(config)
                 peer_checker = makepeer(ui, url)
                 peer_checker.lookup('tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
                         cleaned_uri)
             raise exceptions.URLError(e)(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
         log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
     def diff(
             self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
             context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             match_filter = match(file_filter[0], '', [file_filter[1]])
         else:
             match_filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
 
         try:
             return "".join(patch.diff(
                 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
         except RepoLookupError as e:
             raise exceptions.LookupException(e)()
 
     @reraise_safe_exceptions
-    def file_history(self, wire, revision, path, limit):
+    def node_history(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
 
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         def history_iter():
             limit_rev = fctx.rev()
             for obj in reversed(list(fctx.filelog())):
                 obj = fctx.filectx(obj)
                 if limit_rev >= obj.rev():
                     yield obj
 
         history = []
         for cnt, obj in enumerate(history_iter()):
             if limit and cnt >= limit:
                 break
             history.append(hex(obj.node()))
 
         return [x for x in history]
 
     @reraise_safe_exceptions
-    def file_history_untill(self, wire, revision, path, limit):
+    def node_history_untill(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         file_log = list(fctx.filelog())
         if limit:
             # Limit to the last n items
             file_log = file_log[-limit:]
 
         return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
 
     @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         result = []
         for i, annotate_obj in enumerate(fctx.annotate(), 1):
             ln_no = i
             sha = hex(annotate_obj.fctx.node())
             content = annotate_obj.text
             result.append((ln_no, sha, content))
         return result
 
     @reraise_safe_exceptions
     def fctx_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.data()
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.flags()
 
     @reraise_safe_exceptions
     def fctx_size(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.size()
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
         repo = self._factory.repo(wire)
         revs = repo.filtered(name).changelog.index
         return map(lambda x: hex(x[7]), revs)[:-1]
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_bool(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configbool(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_list(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configlist(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return largefiles.lfutil.isstandin(path)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
 
         repo = self._factory.repo(wire)
 
         if isinstance(revision, int):
             # NOTE(marcink):
             # since Mercurial doesn't support indexes properly
             # we need to shift accordingly by one to get proper index, e.g
             # repo[-1] => repo[-2]
             # repo[0]  => repo[-1]
             # repo[1]  => repo[2] we also never call repo[0] because
             # it's actually second commit
             if revision <= 0:
                 revision = revision + -1
             else:
                 revision = revision + 1
 
         try:
             ctx = repo[revision]
         except RepoLookupError as e:
             raise exceptions.LookupException(e)(revision)
         except LookupError as e:
             raise exceptions.LookupException(e)(e.name)
 
         if not both:
             return ctx.hex()
 
         ctx = repo[ctx.hex()]
         return ctx.hex(), ctx.rev()
 
     @reraise_safe_exceptions
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         # Disable any prompts for this repo
         repo.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         remote = peer(repo, {}, url)
         # Disable any prompts for this remote
         remote.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
 
         return exchange.pull(
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url):
         if not self.check_url(url, wire['config']):
             return
 
         repo = self._factory.repo(wire)
 
         # Disable any prompts for this repo
         repo.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         bookmarks = dict(repo._bookmarks).keys()
         remote = peer(repo, {}, url)
         # Disable any prompts for this remote
         remote.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         return exchange.push(
             repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = repo[rev]
         return ctx.rev()
 
     @reraise_safe_exceptions
     def rev_range(self, wire, filter):
         repo = self._factory.repo(wire)
         revisions = [rev for rev in revrange(repo, filter)]
         return revisions
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         other_path = kwargs.pop('other_path', None)
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.unionrepository(baseui, other_path, wire["path"])
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         hgext_strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui.setconfig('ui', 'quiet', 'false')
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
         baseui.write = write
 
         repo.ui = baseui
         verify.verify(repo)
         return output.getvalue()
 
     @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user,
             tag_time, tag_timezone):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         node = ctx.node()
 
         date = (tag_time, tag_timezone)
         try:
             hg_tag.tag(repo, name, node, message, local, user, date)
         except Abort as e:
             log.exception("Tag operation aborted")
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
     def tags(self, wire):
         repo = self._factory.repo(wire)
         return repo.tags()
 
     @reraise_safe_exceptions
     def update(self, wire, node=None, clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
                  hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
         if bookmark:
             opts['bookmark'] = bookmark
         if branch:
             opts['branch'] = branch
         if revision:
             opts['rev'] = revision
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [branch]
         else:
             args = []
         commands.heads(baseui, repo, template='{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
         a = changelog.ancestor(lookup(revision1), lookup(revision2))
         return hex(a)
 
     @reraise_safe_exceptions
     def push(self, wire, revisions, dest_path, hooks=True,
              push_branches=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
         commands.push(baseui, repo, dest=dest_path, rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
 
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username, close_branch=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'username', username)
         commands.commit(baseui, repo, message=message, close_branch=close_branch)
 
     @reraise_safe_exceptions
     def rebase(self, wire, source=None, dest=None, abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
         rebase.rebase(
             baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         # we don't need any special hooks for Mercurial
         pass

vcsserver/hook_utils/__init__.py

@@ -1,154 +1,154 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import re
 import os
 import sys
 import datetime
 import logging
 import pkg_resources
 
 import vcsserver
 
 log = logging.getLogger(__name__)
 
 
 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
     """
     Creates a RhodeCode hook inside a git repository
 
     :param repo_path: path to repository
     :param executable: binary executable to put in the hooks
     :param force_create: Create even if same name hook exists
     """
     executable = executable or sys.executable
     hooks_path = os.path.join(repo_path, 'hooks')
     if not bare:
         hooks_path = os.path.join(repo_path, '.git', 'hooks')
     if not os.path.isdir(hooks_path):
-        os.makedirs(hooks_path, mode=0777)
+        os.makedirs(hooks_path, mode=0o777)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
     tmpl_pre = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
 
     path = ''  # not used for now
     timestamp = datetime.datetime.utcnow().isoformat()
 
     for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
         log.debug('Installing git hook in repo %s', repo_path)
         _hook_file = os.path.join(hooks_path, '%s-receive' % h_type)
         _rhodecode_hook = check_rhodecode_hook(_hook_file)
 
         if _rhodecode_hook or force_create:
             log.debug('writing git %s hook file at %s !', h_type, _hook_file)
             try:
                 with open(_hook_file, 'wb') as f:
                     template = template.replace(
                         '_TMPL_', vcsserver.__version__)
                     template = template.replace('_DATE_', timestamp)
                     template = template.replace('_ENV_', executable)
                     template = template.replace('_PATH_', path)
                     f.write(template)
-                os.chmod(_hook_file, 0755)
+                os.chmod(_hook_file, 0o755)
             except IOError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
 
     return True
 
 
 def install_svn_hooks(repo_path, executable=None, force_create=False):
     """
     Creates RhodeCode hooks inside a svn repository
 
     :param repo_path: path to repository
     :param executable: binary executable to put in the hooks
     :param force_create: Create even if same name hook exists
     """
     executable = executable or sys.executable
     hooks_path = os.path.join(repo_path, 'hooks')
     if not os.path.isdir(hooks_path):
-        os.makedirs(hooks_path, mode=0777)
+        os.makedirs(hooks_path, mode=0o777)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
     tmpl_pre = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
 
     path = ''  # not used for now
     timestamp = datetime.datetime.utcnow().isoformat()
 
     for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
         log.debug('Installing svn hook in repo %s', repo_path)
         _hook_file = os.path.join(hooks_path, '%s-commit' % h_type)
         _rhodecode_hook = check_rhodecode_hook(_hook_file)
 
         if _rhodecode_hook or force_create:
             log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
 
             try:
                 with open(_hook_file, 'wb') as f:
                     template = template.replace(
                         '_TMPL_', vcsserver.__version__)
                     template = template.replace('_DATE_', timestamp)
                     template = template.replace('_ENV_', executable)
                     template = template.replace('_PATH_', path)
 
                     f.write(template)
-                os.chmod(_hook_file, 0755)
+                os.chmod(_hook_file, 0o755)
             except IOError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
 
     return True
 
 
 def check_rhodecode_hook(hook_path):
     """
     Check if the hook was created by RhodeCode
     """
     if not os.path.exists(hook_path):
         return True
 
     log.debug('hook exists, checking if it is from rhodecode')
     hook_content = read_hook_content(hook_path)
     matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
     if matches:
         try:
             version = matches.groups()[0]
             log.debug('got version %s from hooks.', version)
             return True
         except Exception:
             log.exception("Exception while reading the hook version.")
 
     return False
 
 
 def read_hook_content(hook_path):
     with open(hook_path, 'rb') as f:
         content = f.read()
     return content

vcsserver/hooks.py

@@ -1,702 +1,711 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import logging
 import collections
 import importlib
 import base64
 
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import simplejson as json
 
 from vcsserver import exceptions, subprocessio, settings
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         try:
             connection.request('POST', '/', body)
         except Exception:
             log.error('Connection failed on %s', connection)
             raise
         response = connection.getresponse()
-        return json.loads(response.read())
+
+        response_data = response.read()
+
+        try:
+            return json.loads(response_data)
+        except Exception:
+            log.exception('Failed to decode hook response json data. '
+                          'response_code:%s, raw_data:%s',
+                          response.status, response_data)
+            raise
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 class SvnMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to svn clients."""
 
     def __init__(self, stderr=None):
         # SVN needs data sent to stderr for back-to-client messaging
         self.stderr = stderr or sys.stderr
 
     def write(self, message):
         self.stderr.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
     if 'hooks_uri' in extras:
         protocol = extras.get('hooks_protocol')
         return HooksHttpClient(extras['hooks_uri'])
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks_client = _get_hooks_client(extras)
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
 
     _handle_exception(result)
     writer.write(result['output'])
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node, check_heads=False):
 
     commits = []
     revs = []
     start = repo[node].rev()
     end = len(repo)
     for rev in range(start, end):
         revs.append(rev)
         ctx = repo[rev]
         commit_id = mercurial.node.hex(ctx.node())
         branch = ctx.branch()
         commits.append((commit_id, branch))
 
     parent_heads = []
     if check_heads:
         parent_heads = _check_heads(repo, start, end, revs)
     return commits, parent_heads
 
 
 def _check_heads(repo, start, end, commits):
     changelog = repo.changelog
     parents = set()
 
     for new_rev in commits:
         for p in changelog.parentrevs(new_rev):
             if p == mercurial.node.nullrev:
                 continue
             if p < start:
                 parents.add(p)
 
     for p in parents:
         branch = repo[p].branch()
         # The heads descending from that parent, on the same branch
         parent_heads = set([p])
         reachable = set([p])
         for x in xrange(p + 1, end):
             if repo[x].branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
                 if pp in reachable:
                     reachable.add(x)
                     parent_heads.discard(pp)
                     parent_heads.add(x)
         # More than one head? Suggest merging
         if len(parent_heads) > 1:
             return list(parent_heads)
 
     return []
 
 
 def _get_git_env():
     env = {}
     for k, v in os.environ.items():
         if k.startswith('GIT'):
             env[k] = v
 
     # serialized version
     return [(k, v) for k, v in env.items()]
 
 
 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
     env = {}
     for k, v in os.environ.items():
         if k.startswith('HG'):
             env[k] = v
 
     env['HG_NODE'] = old_rev
     env['HG_NODE_LAST'] = new_rev
     env['HG_TXNID'] = txnid
     env['HG_PENDING'] = repo_path
 
     return [(k, v) for k, v in env.items()]
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook
     """
     extras = _extras_from_ui(ui)
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
     if node and kwargs.get('hooktype') == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
             old_rev = kwargs.get('node_last') or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
         for push_ref in rev_data:
             push_ref['multiple_heads'] = _heads
 
             repo_path = os.path.join(
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
                 new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
                 repo_path=repo_path)
 
     extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook for SSH
     """
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook
     """
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_branches'):
         bookmarks = ui._rc_pushkey_branches
 
     extras['hook_type'] = kwargs.get('hooktype', 'post_push')
     extras['commit_ids'] = commit_ids
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook for SSH
     """
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
     if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_branches = repo[kwargs['key']].bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 # NOTE(marcink):
                 # we're unable to tell total_commits for git at this point
                 # but we set the variable for consistency with GIT
                 'total_commits': -1,
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     empty_commit_id = '0' * 40
 
     detect_force_push = extras.get('detect_force_push')
 
     for push_ref in rev_data:
         # store our git-env which holds the temp store
         push_ref['git_env'] = _get_git_env()
         push_ref['pruned_sha'] = ''
         if not detect_force_push:
             # don't check for forced-push when we don't need to
             continue
 
         type_ = push_ref['type']
         new_branch = push_ref['old_rev'] == empty_commit_id
         if type_ == 'heads' and not new_branch:
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
             cmd = [settings.GIT_EXECUTABLE, 'rev-list',
                    old_rev, '^{}'.format(new_rev)]
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push
             # was used
             if stdout:
                 push_ref['pruned_sha'] = stdout.splitlines()
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
                 # starting new branch case
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 # Fix up head revision if needed
                 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
                 try:
                     subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
                     cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
                     print("Setting default branch to %s" % push_ref['name'])
                     subprocessio.run_command(cmd, env=os.environ.copy())
 
                 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
                        '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 heads = stdout
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
                 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
                        '--pretty=format:%H', '--', push_ref['new_rev'],
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE, 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['hook_type'] = 'post_receive'
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())
 
 
 def _get_extras_from_txn_id(path, txn_id):
     extras = {}
     try:
         cmd = ['svnlook', 'pget',
                '-t', txn_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from txn_id')
 
     return extras
 
 
 def _get_extras_from_commit_id(commit_id, path):
     extras = {}
     try:
         cmd = ['svnlook', 'pget',
                '-r', commit_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from commit_id')
 
     return extras
 
 
 def svn_pre_commit(repo_path, commit_data, env):
     path, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_txn_id(path, txn_id)
         if not extras:
             return 0
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = []
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'total_commits': 1,
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     return _call_hook('pre_push', extras, SvnMessageWriter())
 
 
 def svn_post_commit(repo_path, commit_data, env):
     """
     commit_data is path, rev, txn_id
     """
     path, commit_id, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_commit_id(commit_id, path)
         if not extras:
             return 0
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
         'total_commits': 1,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, SvnMessageWriter())
         except Exception:
             pass
 
     return _call_hook('post_push', extras, SvnMessageWriter())

vcsserver/http_main.py

@@ -1,598 +1,607 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import sys
 import base64
 import locale
 import logging
 import uuid
 import wsgiref.util
 import traceback
 import tempfile
 from itertools import chain
 
 import simplejson as json
 import msgpack
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.compat import configparser
 
 
 log = logging.getLogger(__name__)
 
 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
 
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
     log.error(
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
 
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
 
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 
 def _is_request_chunked(environ):
     stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
     return stream
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
     return settings[name]
 
 
 def _bool_setting(settings, name, default):
     input_val = settings.get(name, default)
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
     return settings[name]
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     # Otherwise we assume it uses pyramids space/newline separation.
     settings[name] = aslist(raw_value)
     return settings[name]
 
 
 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
 
     if default_when_empty and not value:
         # use default value when value is empty
         value = default
 
     if lower:
         value = value.lower()
     settings[name] = value
     return settings[name]
 
 
 class VCS(object):
     def __init__(self, locale=None, cache_config=None):
         self.locale = locale
         self.cache_config = cache_config
         self._configure_locale()
 
         if GitFactory and GitRemote:
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
 
             # hg factory is used for svn url validation
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.info("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
+def not_found(request):
+    return {'status': '404 NOT FOUND'}
+
+
+class VCSViewPredicate(object):
+    def __init__(self, val, config):
+        self.remotes = val
+
+    def text(self):
+        return 'vcs view method = %s' % (self.remotes.keys(),)
+
+    phash = text
+
+    def __call__(self, context, request):
+        """
+        View predicate that returns true if given backend is supported by
+        defined remotes.
+        """
+        backend = request.matchdict.get('backend')
+        return backend in self.remotes
+
+
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
         self._sanitize_settings_and_apply_defaults(settings)
 
         self.config = Configurator(settings=settings)
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
 
         locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale=locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
         self._configure_settings(global_config, settings)
         self._configure()
 
     def _configure_settings(self, global_config, app_settings):
         """
         Configure the settings module.
         """
         settings_merged = global_config.copy()
         settings_merged.update(app_settings)
 
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
         binary_dir = app_settings.get('core.binary_dir', None)
         if binary_dir:
             settings.BINARY_DIR = binary_dir
 
         # Store the settings to make them available to other modules.
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
     def _sanitize_settings_and_apply_defaults(self, settings):
         temp_store = tempfile.gettempdir()
         default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
         # save default, cache dir, and use it for all backends later.
         default_cache_dir = _string_setting(
             settings,
             'cache_dir',
             default_cache_dir, lower=False, default_when_empty=True)
 
         # ensure we have our dir created
         if not os.path.isdir(default_cache_dir):
-            os.makedirs(default_cache_dir, mode=0755)
+            os.makedirs(default_cache_dir, mode=0o755)
 
         # exception store cache
         _string_setting(
             settings,
             'exception_tracker.store_path',
             temp_store, lower=False, default_when_empty=True)
 
         # repo_object cache
         _string_setting(
             settings,
             'rc_cache.repo_object.backend',
             'dogpile.cache.rc.memory_lru')
         _int_setting(
             settings,
             'rc_cache.repo_object.expiration_time',
             300)
         _int_setting(
             settings,
             'rc_cache.repo_object.max_size',
             1024)
 
     def _configure(self):
-        self.config.add_renderer(
-            name='msgpack',
-            factory=self._msgpack_renderer_factory)
+        self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
         self.config.add_route('vcs', '/{backend}')
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
-        self.config.add_view(
-            self.status_view, route_name='status', renderer='json')
-        self.config.add_view(
-            self.service_view, route_name='service', renderer='msgpack')
+        self.config.add_view(self.status_view, route_name='status', renderer='json')
+        self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
-        self.config.add_view(
-            self.vcs_view, route_name='vcs', renderer='msgpack',
-            custom_predicates=[self.is_vcs_view])
+        self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
+                             vcs_view=self._remotes)
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
-        def notfound(request):
-            return {'status': '404 NOT FOUND'}
-        self.config.add_notfound_view(notfound, renderer='json')
+        self.config.add_view_predicate('vcs_view', VCSViewPredicate)
+
+        self.config.add_notfound_view(not_found, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.RequestWrapperTween',
         )
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def vcs_view(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
         params = payload.get('params')
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         context_uid = None
 
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
 
         log.debug('method called:%s with kwargs:%s context_uid: %s',
                   method, kwargs, context_uid)
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             exc_info = list(sys.exc_info())
             exc_type, exc_value, exc_traceback = exc_info
 
             org_exc = getattr(e, '_org_exc', None)
             org_exc_name = None
             if org_exc:
                 org_exc_name = org_exc.__class__.__name__
                 # replace our "faked" exception with our org
                 exc_info[0] = org_exc.__class__
                 exc_info[1] = org_exc
 
             store_exception(id(exc_info), exc_info)
 
             tb_info = ''.join(
                 traceback.format_exception(exc_type, exc_value, exc_traceback))
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload.get('id'),
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'type': type_
                 }
             }
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload.get('id'),
                 'result': resp
             }
 
         return resp
 
     def status_view(self, request):
         import vcsserver
         return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
                 'pid': os.getpid()}
 
     def service_view(self, request):
         import vcsserver
 
         payload = msgpack.unpackb(request.body, use_list=True)
 
         try:
             path = self.global_config['__file__']
             config = configparser.ConfigParser()
             config.read(path)
             parsed_ini = config
             if parsed_ini.has_section('server:main'):
                 parsed_ini = dict(parsed_ini.items('server:main'))
         except Exception:
             log.exception('Failed to read .ini file for display')
             parsed_ini = {}
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
                 config=parsed_ini,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             value = msgpack.packb(value)
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
             return value
         return _render
 
     def set_env_from_config(self, environ, config):
         dict_conf = {}
         try:
             for elem in config:
                 if elem[0] == 'rhodecode':
                     dict_conf = json.loads(elem[2])
                     break
         except Exception:
             log.exception('Failed to fetch SCM CONFIG')
             return
 
         username = dict_conf.get('username')
         if username:
             environ['REMOTE_USER'] = username
             # mercurial specific, some extension api rely on this
             environ['HGUSER'] = username
 
         ip = dict_conf.get('ip')
         if ip:
             environ['REMOTE_HOST'] = ip
 
         if _is_request_chunked(environ):
             # set the compatibility flag for webob
             environ['wsgi.input_terminated'] = True
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
 
                 path = environ['PATH_INFO']
                 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
                 log.debug(
                     'LFS: Detecting if request `%s` is LFS server path based '
                     'on content type:`%s`, is_lfs:%s',
                     path, content_type, is_lfs_request)
 
                 if not is_lfs_request:
                     # fallback detection by path
                     if GIT_LFS_PROTO_PAT.match(path):
                         is_lfs_request = True
                     log.debug(
                         'LFS: fallback detection by path of: `%s`, is_lfs:%s',
                         path, is_lfs_request)
 
                 if is_lfs_request:
                     app = scm_app.create_git_lfs_wsgi_app(
                         repo_path, repo_name, config)
                 else:
                     app = scm_app.create_git_wsgi_app(
                         repo_path, repo_name, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
 
                 return app(environ, start_response)
 
             return _git_stream
 
-    def is_vcs_view(self, context, request):
-        """
-        View predicate that returns true if given backend is supported by
-        defined remotes.
-        """
-        backend = request.matchdict.get('backend')
-        return backend in self._remotes
-
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
         if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
             return HTTPRepoBranchProtected(title=exception.message)
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
 
         traceback_info = 'unavailable'
         if request.exc_info:
             exc_type, exc_value, exc_tb = request.exc_info
             traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
 
         log.error(
             'error occurred handling this request for path: %s, \n tb: %s',
             request.path, traceback_info)
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
     app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/tests/test_subprocessio.py

@@ -1,122 +1,132 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 
 import pytest
 
 from vcsserver import subprocessio
 
 
 @pytest.fixture(scope='module')
 def environ():
     """Delete coverage variables, as they make the tests fail."""
     env = dict(os.environ)
     for key in env.keys():
         if key.startswith('COV_CORE_'):
             del env[key]
 
     return env
 
 
 def _get_python_args(script):
-    return [sys.executable, '-c',
-            'import sys; import time; import shutil; ' + script]
+    return [sys.executable, '-c', 'import sys; import time; import shutil; ' + script]
 
 
 def test_raise_exception_on_non_zero_return_code(environ):
     args = _get_python_args('sys.exit(1)')
     with pytest.raises(EnvironmentError):
         list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
 
 def test_does_not_fail_on_non_zero_return_code(environ):
     args = _get_python_args('sys.exit(1)')
-    output = ''.join(subprocessio.SubprocessIOChunker(
-        args, shell=False, fail_on_return_code=False, env=environ))
+    output = ''.join(
+        subprocessio.SubprocessIOChunker(
+            args, shell=False, fail_on_return_code=False, env=environ
+        )
+    )
 
     assert output == ''
 
 
 def test_raise_exception_on_stderr(environ):
     args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
     with pytest.raises(EnvironmentError) as excinfo:
         list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
     assert 'exited due to an error:\nX' in str(excinfo.value)
 
 
 def test_does_not_fail_on_stderr(environ):
     args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
-    output = ''.join(subprocessio.SubprocessIOChunker(
-        args, shell=False, fail_on_stderr=False, env=environ))
+    output = ''.join(
+        subprocessio.SubprocessIOChunker(
+            args, shell=False, fail_on_stderr=False, env=environ
+        )
+    )
 
     assert output == ''
 
 
[e-mail]('size', [1, 10**5])
[e-mail]('size', [1, 10 ** 5])
 def test_output_with_no_input(size, environ):
-    print type(environ)
+    print(type(environ))
     data = 'X'
     args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size))
-    output = ''.join(subprocessio.SubprocessIOChunker(
-        args, shell=False, env=environ))
+    output = ''.join(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
     assert output == data * size
 
 
[e-mail]('size', [1, 10**5])
[e-mail]('size', [1, 10 ** 5])
 def test_output_with_no_input_does_not_fail(size, environ):
     data = 'X'
-    args = _get_python_args(
-        'sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
-    output = ''.join(subprocessio.SubprocessIOChunker(
-        args, shell=False, fail_on_return_code=False, env=environ))
+    args = _get_python_args('sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
+    output = ''.join(
+        subprocessio.SubprocessIOChunker(
+            args, shell=False, fail_on_return_code=False, env=environ
+        )
+    )
 
-    print len(data * size), len(output)
+    print("{} {}".format(len(data * size), len(output)))
     assert output == data * size
 
 
[e-mail]('size', [1, 10**5])
[e-mail]('size', [1, 10 ** 5])
 def test_output_with_input(size, environ):
     data = 'X' * size
     inputstream = io.BytesIO(data)
     # This acts like the cat command.
     args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
-    output = ''.join(subprocessio.SubprocessIOChunker(
-        args, shell=False, inputstream=inputstream, env=environ))
+    output = ''.join(
+        subprocessio.SubprocessIOChunker(
+            args, shell=False, inputstream=inputstream, env=environ
+        )
+    )
 
-    print len(data), len(output)
+    print("{} {}".format(len(data * size), len(output)))
     assert output == data
 
 
[e-mail]('size', [1, 10**5])
[e-mail]('size', [1, 10 ** 5])
 def test_output_with_input_skipping_iterator(size, environ):
     data = 'X' * size
     inputstream = io.BytesIO(data)
     # This acts like the cat command.
     args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
 
     # Note: assigning the chunker makes sure that it is not deleted too early
     chunker = subprocessio.SubprocessIOChunker(
-        args, shell=False, inputstream=inputstream, env=environ)
+        args, shell=False, inputstream=inputstream, env=environ
+    )
     output = ''.join(chunker.output)
 
-    print len(data), len(output)
+    print("{} {}".format(len(data * size), len(output)))
     assert output == data