release: Merge default into stable for release preparation

.bumpversion.cfg

@@ -1,6 +1,6 b''
 [bumpversion]
-current_version = 4.18.3
+current_version = 4.19.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]
 

.release.cfg

@@ -1,16 +1,14 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.18.3
+state = in_progress
+version = 4.19.0
 

Makefile

@@ -1,23 +1,28 b''
 
-.PHONY: clean test test-clean test-only generate-pkgs
+.PHONY: clean test test-clean test-only generate-pkgs pip-packages
 
+# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
+OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 
 test:
 	make test-clean
 	make test-only
 
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov=vcsserver --cov-report=term-missing --cov-report=html vcsserver
 
 generate-pkgs:
 	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+
+pip-packages:
+	python ${OUTDATED_PACKAGES}

pkgs/overlays.nix

@@ -1,71 +1,71 b''
 self: super: {
 
   # bump GIT version
   git = super.lib.overrideDerivation super.git (oldAttrs: {
     name = "git-2.24.1";
     src = self.fetchurl {
       url = "https://www.kernel.org/pub/software/scm/git/git-2.24.1.tar.xz";
       sha256 = "0ql5z31vgl7b785gwrf00m129mg7zi9pa65n12ij3mpxx3f28gvj";
     };
 
     # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
     patches = [
       ./patches/git/docbook2texi.patch
       ./patches/git/git-sh-i18n.patch
       ./patches/git/ssh-path.patch
       ./patches/git/git-send-email-honor-PATH.patch
       ./patches/git/installCheck-path.patch
     ];
 
   });
 
   libgit2rc = super.lib.overrideDerivation super.libgit2 (oldAttrs: {
     name = "libgit2-0.28.2";
     version = "0.28.2";
 
     src = self.fetchFromGitHub {
         owner = "libgit2";
         repo = "libgit2";
         rev = "v0.28.2";
         sha256 = "0cm8fvs05rj0baigs2133q5a0sm3pa234y8h6hmwhl2bz9xq3k4b";
     };
 
     cmakeFlags = [ "-DTHREADSAFE=ON" "-DUSE_HTTPS=no"];
 
     buildInputs = [
         super.zlib
         super.libssh2
         super.openssl
         super.curl
     ];
 
 
   });
 
   # Override subversion derivation to
   #  - activate python bindings
   subversion =
   let
     subversionWithPython = super.subversion.override {
       httpSupport = true;
       pythonBindings = true;
       python = self.python27Packages.python;
     };
   in
     super.lib.overrideDerivation subversionWithPython (oldAttrs: {
-      name = "subversion-1.12.2";
+      name = "subversion-1.13.0";
       src = self.fetchurl {
-        url = "https://archive.apache.org/dist/subversion/subversion-1.12.2.tar.gz";
-        sha256 = "1wr1pklnq67xdzmf237zj6l1hg43yshfkbxvpvd5sv6r0dk7v4pl";
+        url = "https://archive.apache.org/dist/subversion/subversion-1.13.0.tar.gz";
+        sha256 = "0cb9p7f5hg0l4k32hz8vmvy2r45igchq5sh4m366za5q0c649bfs";
       };
 
       ## use internal lz4/utf8proc because it is stable and shipped with SVN
       configureFlags = oldAttrs.configureFlags ++ [
         " --with-lz4=internal"
         " --with-utf8proc=internal"
       ];
 
   });
 
 
 }

pkgs/python-packages.nix

@@ -1,1090 +1,1090 b''
 # Generated by pip2nix 0.8.0.dev1
 # See https://github.com/johbo/pip2nix
 
 { pkgs, fetchurl, fetchgit, fetchhg }:
 
 self: super: {
   "atomicwrites" = super.buildPythonPackage {
     name = "atomicwrites-1.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/0f/cd484ac8820fed363b374af30049adc8fd13065720fd4f4c6be8a2309da7/atomicwrites-1.3.0.tar.gz";
       sha256 = "19ngcscdf3jsqmpcxn6zl5b6anmsajb6izp1smcd1n02midl9abm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "attrs" = super.buildPythonPackage {
     name = "attrs-19.3.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/98/c3/2c227e66b5e896e15ccdae2e00bbc69aa46e9a8ce8869cc5fa96310bf612/attrs-19.3.0.tar.gz";
       sha256 = "0wky4h28n7xnr6xv69p9z6kv8bzn50d10c3drmd9ds8gawbcxdzp";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "backports.shutil-get-terminal-size" = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "beautifulsoup4" = super.buildPythonPackage {
     name = "beautifulsoup4-4.6.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz";
       sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "cffi" = super.buildPythonPackage {
     name = "cffi-1.12.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pycparser"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/93/1a/ab8c62b5838722f29f3daffcc8d4bd61844aa9b5f437341cc890ceee483b/cffi-1.12.3.tar.gz";
       sha256 = "0x075521fxwv0mfp4cqzk7lvmw4n94bjw601qkcv314z5s182704";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "configobj" = super.buildPythonPackage {
     name = "configobj-5.0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626";
       sha256 = "0kqfrdfr14mw8yd8qwq14dv2xghpkjmd3yjsy8dfcbvpcc17xnxp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "configparser" = super.buildPythonPackage {
     name = "configparser-4.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/4f/48975536bd488d3a272549eb795ac4a13a5f7fcdc8995def77fbef3532ee/configparser-4.0.2.tar.gz";
       sha256 = "1priacxym85yjcf68hh38w55nqswaxp71ryjyfdk222kg9l85ln7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "contextlib2" = super.buildPythonPackage {
     name = "contextlib2-0.6.0.post1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/02/54/669207eb72e3d8ae8b38aa1f0703ee87a0e9f88f30d3c0a47bebdb6de242/contextlib2-0.6.0.post1.tar.gz";
       sha256 = "0bhnr2ac7wy5l85ji909gyljyk85n92w8pdvslmrvc8qih4r1x01";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "coverage" = super.buildPythonPackage {
     name = "coverage-4.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz";
       sha256 = "0p0j4di6h8k6ica7jwwj09azdcg4ycxq60i9qsskmsg94cd9yzg0";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "decorator" = super.buildPythonPackage {
     name = "decorator-4.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz";
       sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   "dogpile.cache" = super.buildPythonPackage {
     name = "dogpile.cache-0.9.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."decorator"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ac/6a/9ac405686a94b7f009a20a50070a5786b0e1aedc707b88d40d0c4b51a82e/dogpile.cache-0.9.0.tar.gz";
       sha256 = "0sr1fn6b4k5bh0cscd9yi8csqxvj4ngzildav58x5p694mc86j5k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dogpile.core" = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dulwich" = super.buildPythonPackage {
     name = "dulwich-0.13.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz";
       sha256 = "0f1jwvrh549c4rgavkn3wizrch904s73s4fmrxykxy9cw8s57lwf";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "enum34" = super.buildPythonPackage {
-    name = "enum34-1.1.6";
+    name = "enum34-1.1.10";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
-      sha256 = "1cgm5ng2gcfrkrm3hc22brl6chdmv67b9zvva9sfs7gn7dwc9n4a";
+      url = "https://files.pythonhosted.org/packages/11/c4/2da1f4952ba476677a42f25cd32ab8aaf0e1c0d0e00b89822b835c7e654c/enum34-1.1.10.tar.gz";
+      sha256 = "0j7ji699fwswm4vg6w1v07fkbf8dkzdm6gfh88jvs5nqgr3sgrnc";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "funcsigs" = super.buildPythonPackage {
     name = "funcsigs-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz";
       sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7";
     };
     meta = {
       license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "gevent" = super.buildPythonPackage {
-    name = "gevent-1.4.0";
+    name = "gevent-1.5.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."greenlet"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/ed/27/6c49b70808f569b66ec7fac2e78f076e9b204db9cf5768740cff3d5a07ae/gevent-1.4.0.tar.gz";
-      sha256 = "1lchr4akw2jkm5v4kz7bdm4wv3knkfhbfn9vkkz4s5yrkcxzmdqy";
+      url = "https://files.pythonhosted.org/packages/5a/79/2c63d385d017b5dd7d70983a463dfd25befae70c824fedb857df6e72eff2/gevent-1.5.0.tar.gz";
+      sha256 = "0aac3d4vhv5n4rsb6cqzq0d1xx9immqz4fmpddw35yxkwdc450dj";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gprof2dot" = super.buildPythonPackage {
     name = "gprof2dot-2017.9.19";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz";
       sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f";
     };
     meta = {
       license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ];
     };
   };
   "greenlet" = super.buildPythonPackage {
     name = "greenlet-0.4.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz";
       sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gunicorn" = super.buildPythonPackage {
     name = "gunicorn-19.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz";
       sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "hg-evolve" = super.buildPythonPackage {
     name = "hg-evolve-9.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/20/36/5a6655975aa0c663be91098d31a0b24841acad44fe896aa2bdee77c6b883/hg-evolve-9.1.0.tar.gz";
       sha256 = "1mna81cmzxxn7s2nwz3g1xgdjlcc1axkvfmwg7gjqghwn3pdraps";
     };
     meta = {
       license = [ { fullName = "GPLv2+"; } ];
     };
   };
   "hgsubversion" = super.buildPythonPackage {
     name = "hgsubversion-1.9.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."mercurial"
       self."subvertpy"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a3/53/6d205e641f3e09abcf1ddaed66e5e4b20da22d0145566d440a02c9e35f0d/hgsubversion-1.9.3.tar.gz";
       sha256 = "0nymcjlch8c4zjbncrs30p2nrbylsf25g3h6mr0zzzxr141h3sig";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   "hupper" = super.buildPythonPackage {
-    name = "hupper-1.9.1";
+    name = "hupper-1.10.2";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/09/3a/4f215659f31eeffe364a984dba486bfa3907bfcc54b7013bdfe825cebb5f/hupper-1.9.1.tar.gz";
-      sha256 = "0pyg879fv9mbwlnbzw2a3234qqycqs9l97h5mpkmk0bvxhi2471v";
+      url = "https://files.pythonhosted.org/packages/41/24/ea90fef04706e54bd1635c05c50dc9cf87cda543c59303a03e7aa7dda0ce/hupper-1.10.2.tar.gz";
+      sha256 = "0am0p6g5cz6xmcaf04xq8q6dzdd9qz0phj6gcmpsckf2mcyza61q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "importlib-metadata" = super.buildPythonPackage {
-    name = "importlib-metadata-0.23";
+    name = "importlib-metadata-1.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."zipp"
+      self."pathlib2"
       self."contextlib2"
       self."configparser"
-      self."pathlib2"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/5d/44/636bcd15697791943e2dedda0dbe098d8530a38d113b202817133e0b06c0/importlib_metadata-0.23.tar.gz";
-      sha256 = "09mdqdfv5rdrwz80jh9m379gxmvk2vhjfz0fg53hid00icvxf65a";
+      url = "https://files.pythonhosted.org/packages/b4/1b/baab42e3cd64c9d5caac25a9d6c054f8324cdc38975a44d600569f1f7158/importlib_metadata-1.6.0.tar.gz";
+      sha256 = "07icyggasn38yv2swdrd8z6i0plazmc9adavsdkbqqj91j53ll9l";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "ipdb" = super.buildPythonPackage {
-    name = "ipdb-0.12";
+    name = "ipdb-0.13.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."ipython"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/6d/43/c3c2e866a8803e196d6209595020a4a6db1a3c5d07c01455669497ae23d0/ipdb-0.12.tar.gz";
-      sha256 = "1khr2n7xfy8hg65kj1bsrjq9g7656pp0ybfa8abpbzpdawji3qnw";
+      url = "https://files.pythonhosted.org/packages/2c/bb/a3e1a441719ebd75c6dac8170d3ddba884b7ee8a5c0f9aefa7297386627a/ipdb-0.13.2.tar.gz";
+      sha256 = "0jcd849rx30y3wcgzsqbn06v0yjlzvb9x3076q0yxpycdwm1ryvp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython" = super.buildPythonPackage {
     name = "ipython-5.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."decorator"
       self."pickleshare"
       self."simplegeneric"
       self."traitlets"
       self."prompt-toolkit"
       self."pygments"
       self."pexpect"
       self."backports.shutil-get-terminal-size"
       self."pathlib2"
       self."pexpect"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython-genutils" = super.buildPythonPackage {
     name = "ipython-genutils-0.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz";
       sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mako" = super.buildPythonPackage {
     name = "mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz";
       sha256 = "0jqa3qfpykyn4fmkn0kh6043sfls7br8i2bsdbccazcvk9cijsd3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "markupsafe" = super.buildPythonPackage {
     name = "markupsafe-1.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz";
       sha256 = "0sqipg4fk7xbixqd8kq6rlkxj664d157bdwbh93farcphf92x1r9";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd3 ];
     };
   };
   "mercurial" = super.buildPythonPackage {
     name = "mercurial-5.1.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/22/39/e1a95f6048aa0785b82f5faad8281ae7320894a635cb4a57e19479639c92/mercurial-5.1.1.tar.gz";
       sha256 = "17z42rfjdkrks4grzgac66nfh285zf1pwxd2zwx1p71pw2jqpz1m";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "mock" = super.buildPythonPackage {
     name = "mock-3.0.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."funcsigs"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2e/ab/4fe657d78b270aa6a32f027849513b829b41b0f28d9d8d7f8c3d29ea559a/mock-3.0.5.tar.gz";
       sha256 = "1hrp6j0yrx2xzylfv02qa8kph661m6yq4p0mc8fnimch9j4psrc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "OSI Approved :: BSD License"; } ];
     };
   };
   "more-itertools" = super.buildPythonPackage {
     name = "more-itertools-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/dd/26/30fc0d541d9fdf55faf5ba4b0fd68f81d5bd2447579224820ad525934178/more-itertools-5.0.0.tar.gz";
       sha256 = "1r12cm6mcdwdzz7d47a6g4l437xsvapdlgyhqay3i2nrlv03da9q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "msgpack-python" = super.buildPythonPackage {
     name = "msgpack-python-0.5.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz";
       sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "packaging" = super.buildPythonPackage {
-    name = "packaging-19.2";
+    name = "packaging-20.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyparsing"
       self."six"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/5a/2f/449ded84226d0e2fda8da9252e5ee7731bdf14cd338f622dfcd9934e0377/packaging-19.2.tar.gz";
-      sha256 = "0izwlz9h0bw171a1chr311g2y7n657zjaf4mq4rgm8pp9lbj9f98";
+      url = "https://files.pythonhosted.org/packages/65/37/83e3f492eb52d771e2820e88105f605335553fe10422cba9d256faeb1702/packaging-20.3.tar.gz";
+      sha256 = "18xpablq278janh03bai9xd4kz9b0yfp6vflazn725ns9x3jna9w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "pastedeploy" = super.buildPythonPackage {
-    name = "pastedeploy-2.0.1";
+    name = "pastedeploy-2.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/19/a0/5623701df7e2478a68a1b685d1a84518024eef994cde7e4da8449a31616f/PasteDeploy-2.0.1.tar.gz";
-      sha256 = "02imfbbx1mi2h546f3sr37m47dk9qizaqhzzlhx8bkzxa6fzn8yl";
+      url = "https://files.pythonhosted.org/packages/c4/e9/972a1c20318b3ae9edcab11a6cef64308fbae5d0d45ab52c6f8b2b8f35b8/PasteDeploy-2.1.0.tar.gz";
+      sha256 = "16qsq5y6mryslmbp5pn35x4z8z3ndp5rpgl42h226879nrw9hmg7";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pathlib2" = super.buildPythonPackage {
     name = "pathlib2-2.3.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."scandir"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/d8/65c86584e7e97ef824a1845c72bbe95d79f5b306364fa778a3c3e401b309/pathlib2-2.3.5.tar.gz";
       sha256 = "0s4qa8c082fdkb17izh4mfgwrjd1n5pya18wvrbwqdvvb5xs9nbc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pexpect" = super.buildPythonPackage {
-    name = "pexpect-4.7.0";
+    name = "pexpect-4.8.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ptyprocess"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/1c/b1/362a0d4235496cb42c33d1d8732b5e2c607b0129ad5fdd76f5a583b9fcb3/pexpect-4.7.0.tar.gz";
-      sha256 = "1sv2rri15zwhds85a4kamwh9pj49qcxv7m4miyr4jfpfwv81yb4y";
+      url = "https://files.pythonhosted.org/packages/e5/9b/ff402e0e930e70467a7178abb7c128709a30dfb22d8777c043e501bc1b10/pexpect-4.8.0.tar.gz";
+      sha256 = "032cg337h8awydgypz6f4wx848lw8dyrj4zy988x0lyib4ws8rgw";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   "pickleshare" = super.buildPythonPackage {
     name = "pickleshare-0.7.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz";
       sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster" = super.buildPythonPackage {
     name = "plaster-1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz";
       sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster-pastedeploy" = super.buildPythonPackage {
     name = "plaster-pastedeploy-0.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pastedeploy"
       self."plaster"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/69/2d3bc33091249266a1bd3cf24499e40ab31d54dffb4a7d76fe647950b98c/plaster_pastedeploy-0.7.tar.gz";
       sha256 = "1zg7gcsvc1kzay1ry5p699rg2qavfsxqwl17mqxzr0gzw6j9679r";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pluggy" = super.buildPythonPackage {
     name = "pluggy-0.13.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."importlib-metadata"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/04/7a8542bed4b16a65c2714bf76cf5a0b026157da7f75e87cc88774aa10b14/pluggy-0.13.1.tar.gz";
       sha256 = "1c35qyhvy27q9ih9n899f3h4sdnpgq027dbiilly2qb5cvgarchm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "prompt-toolkit" = super.buildPythonPackage {
     name = "prompt-toolkit-1.0.18";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."wcwidth"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c5/64/c170e5b1913b540bf0c8ab7676b21fdd1d25b65ddeb10025c6ca43cccd4c/prompt_toolkit-1.0.18.tar.gz";
       sha256 = "09h1153wgr5x2ny7ds0w2m81n3bb9j8hjb8sjfnrg506r01clkyx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psutil" = super.buildPythonPackage {
-    name = "psutil-5.6.5";
+    name = "psutil-5.7.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/03/9a/95c4b3d0424426e5fd94b5302ff74cea44d5d4f53466e1228ac8e73e14b4/psutil-5.6.5.tar.gz";
-      sha256 = "0isil5jxwwd8awz54qk28rpgjg43i5l6yl70g40vxwa4r4m56lfh";
+      url = "https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz";
+      sha256 = "03jykdi3dgf1cdal9bv4fq9zjvzj9l9bs99gi5ar81sdl5nc2pk8";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ptyprocess" = super.buildPythonPackage {
     name = "ptyprocess-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz";
       sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj";
     };
     meta = {
       license = [  ];
     };
   };
   "py" = super.buildPythonPackage {
     name = "py-1.8.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f1/5a/87ca5909f400a2de1561f1648883af74345fe96349f34f737cdfc94eba8c/py-1.8.0.tar.gz";
       sha256 = "0lsy1gajva083pzc7csj1cvbmminb7b4l6a0prdzyb3fd829nqyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pycparser" = super.buildPythonPackage {
-    name = "pycparser-2.19";
+    name = "pycparser-2.20";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a/pycparser-2.19.tar.gz";
-      sha256 = "1cr5dcj9628lkz1qlwq3fv97c25363qppkmcayqvd05dpy573259";
+      url = "https://files.pythonhosted.org/packages/0f/86/e19659527668d70be91d0369aeaa055b4eb396b0f387a4f92293a20035bd/pycparser-2.20.tar.gz";
+      sha256 = "1w0m3xvlrzq4lkbvd1ngfm8mdw64r1yxy6n7djlw6qj5d0km6ird";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pygit2" = super.buildPythonPackage {
     name = "pygit2-0.28.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."cffi"
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4c/64/88c2a4eb2d22ca1982b364f41ff5da42d61de791d7eb68140e7f8f7eb721/pygit2-0.28.2.tar.gz";
       sha256 = "11kzj5mjkspvplnpdb6bj8dcj6rgmkk986k8hjcklyg5yaxkz32d";
     };
     meta = {
       license = [ { fullName = "GPLv2 with linking exception"; } ];
     };
   };
   "pygments" = super.buildPythonPackage {
     name = "pygments-2.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/ae/26808275fc76bf2832deb10d3a3ed3107bc4de01b85dcccbe525f2cd6d1e/Pygments-2.4.2.tar.gz";
       sha256 = "15v2sqm5g12bqa0c7wikfh9ck2nl97ayizy1hpqhmws5gqalq748";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pyparsing" = super.buildPythonPackage {
-    name = "pyparsing-2.4.5";
+    name = "pyparsing-2.4.7";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/00/32/8076fa13e832bb4dcff379f18f228e5a53412be0631808b9ca2610c0f566/pyparsing-2.4.5.tar.gz";
-      sha256 = "0fk8gsybiw1gm146mkjdjvaajwh20xwvpv4j7syh2zrnpq0j19jc";
+      url = "https://files.pythonhosted.org/packages/c1/47/dfc9c342c9842bbe0036c7f763d2d6686bcf5eb1808ba3e170afdb282210/pyparsing-2.4.7.tar.gz";
+      sha256 = "1hgc8qrbq1ymxbwfbjghv01fm3fbpjwpjwi0bcailxxzhf3yq0y2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pyramid" = super.buildPythonPackage {
     name = "pyramid-1.10.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."hupper"
       self."plaster"
       self."plaster-pastedeploy"
       self."setuptools"
       self."translationstring"
       self."venusian"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c2/43/1ae701c9c6bb3a434358e678a5e72c96e8aa55cf4cb1d2fa2041b5dd38b7/pyramid-1.10.4.tar.gz";
       sha256 = "0rkxs1ajycg2zh1c94xlmls56mx5m161sn8112skj0amza6cn36q";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-mako" = super.buildPythonPackage {
     name = "pyramid-mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."mako"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/63/7b/5e2af68f675071a6bad148c1c393928f0ef5fcd94e95cbf53b89d6471a83/pyramid_mako-1.1.0.tar.gz";
       sha256 = "1qj0m091mnii86j2q1d82yir22nha361rvhclvg3s70z8iiwhrh0";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pytest" = super.buildPythonPackage {
     name = "pytest-4.6.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."py"
       self."six"
       self."packaging"
       self."attrs"
       self."atomicwrites"
       self."pluggy"
       self."importlib-metadata"
       self."wcwidth"
       self."funcsigs"
       self."pathlib2"
       self."more-itertools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/2a/c6/1d1f32f6a5009900521b12e6560fb6b7245b0d4bc3fb771acd63d10e30e1/pytest-4.6.5.tar.gz";
       sha256 = "0iykwwfp4h181nd7rsihh2120b0rkawlw7rvbl19sgfspncr3hwg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-cov" = super.buildPythonPackage {
     name = "pytest-cov-2.7.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/0f/3db7ff86801883b21d5353b258c994b1b8e2abbc804e2273b8d0fd19004b/pytest-cov-2.7.1.tar.gz";
       sha256 = "0filvmmyqm715azsl09ql8hy2x7h286n6d8z5x42a1wpvvys83p0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   "pytest-profiling" = super.buildPythonPackage {
     name = "pytest-profiling-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."pytest"
       self."gprof2dot"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/39/70/22a4b33739f07f1732a63e33bbfbf68e0fa58cfba9d200e76d01921eddbf/pytest-profiling-1.7.0.tar.gz";
       sha256 = "0abz9gi26jpcfdzgsvwad91555lpgdc8kbymicmms8k2fqa8z4wk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-runner" = super.buildPythonPackage {
     name = "pytest-runner-5.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d9/6d/4b41a74b31720e25abd4799be72d54811da4b4d0233e38b75864dcc1f7ad/pytest-runner-5.1.tar.gz";
       sha256 = "0ykfcnpp8c22winj63qzc07l5axwlc9ikl8vn05sc32gv3417815";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-sugar" = super.buildPythonPackage {
     name = "pytest-sugar-0.9.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."termcolor"
       self."packaging"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/55/59/f02f78d1c80f7e03e23177f60624c8106d4f23d124c921df103f65692464/pytest-sugar-0.9.2.tar.gz";
       sha256 = "1asq7yc4g8bx2sn7yy974mhc9ywvaihasjab4inkirdwn9s7mn7w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pytest-timeout" = super.buildPythonPackage {
     name = "pytest-timeout-1.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/13/48/7a166eaa29c1dca6cc253e3ba5773ff2e4aa4f567c1ea3905808e95ac5c1/pytest-timeout-1.3.3.tar.gz";
       sha256 = "1cczcjhw4xx5sjkhxlhc5c1bkr7x6fcyx12wrnvwfckshdvblc2a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   "redis" = super.buildPythonPackage {
-    name = "redis-3.3.11";
+    name = "redis-3.4.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/06/ca/00557c74279d2f256d3c42cabf237631355f3a132e4c74c2000e6647ad98/redis-3.3.11.tar.gz";
-      sha256 = "1hicqbi5xl92hhml82awrr2rxl9jar5fp8nbcycj9qgmsdwc43wd";
+      url = "https://files.pythonhosted.org/packages/ef/2e/2c0f59891db7db087a7eeaa79bc7c7f2c039e71a2b5b0a41391e9d462926/redis-3.4.1.tar.gz";
+      sha256 = "07yaj0j9fs7xdkg5bg926fa990khyigjbp31si8ai20vj8sv7kqd";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "repoze.lru" = super.buildPythonPackage {
     name = "repoze.lru-0.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz";
       sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "rhodecode-vcsserver" = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.18.3";
+    name = "rhodecode-vcsserver-4.19.0";
     buildInputs = [
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
       self."configobj"
     ];
     doCheck = true;
     propagatedBuildInputs = [
       self."configobj"
       self."dogpile.cache"
       self."dogpile.core"
       self."decorator"
       self."dulwich"
       self."hgsubversion"
       self."hg-evolve"
       self."mako"
       self."markupsafe"
       self."mercurial"
       self."msgpack-python"
       self."pastedeploy"
       self."pyramid"
       self."pyramid-mako"
       self."pygit2"
       self."repoze.lru"
       self."redis"
       self."simplejson"
       self."subprocess32"
       self."subvertpy"
       self."six"
       self."translationstring"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."gevent"
       self."greenlet"
       self."gunicorn"
       self."waitress"
       self."ipdb"
       self."ipython"
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
     ];
     src = ./.;
     meta = {
       license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
     };
   };
   "scandir" = super.buildPythonPackage {
     name = "scandir-1.10.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz";
       sha256 = "1bkqwmf056pkchf05ywbnf659wqlp6lljcdb0y88wr9f0vv32ijd";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
     };
   };
   "setproctitle" = super.buildPythonPackage {
     name = "setproctitle-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz";
       sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "setuptools" = super.buildPythonPackage {
-    name = "setuptools-44.0.0";
+    name = "setuptools-44.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/b0/f3/44da7482ac6da3f36f68e253cb04de37365b3dba9036a3c70773b778b485/setuptools-44.0.0.zip";
-      sha256 = "025h5cnxcmda1893l6i12hrwdvs1n8r31qs6q4pkif2v7rrggfp5";
+      url = "https://files.pythonhosted.org/packages/ed/7b/bbf89ca71e722b7f9464ebffe4b5ee20a9e5c9a555a56e2d3914bb9119a6/setuptools-44.1.0.zip";
+      sha256 = "1jja896zvd1ppccnjbhkgagxbwchgq6vfamp6qn1hvywq6q9cjkr";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "simplejson" = super.buildPythonPackage {
     name = "simplejson-3.16.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz";
       sha256 = "19cws1syk8jzq2pw43878dv6fjkb0ifvjpx0i9aajix6kc9jkwxi";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   "six" = super.buildPythonPackage {
     name = "six-1.11.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
       sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "subprocess32" = super.buildPythonPackage {
     name = "subprocess32-3.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz";
       sha256 = "17f7mvwx2271s1wrl0qac3wjqqnrqag866zs3qc8v5wp0k43fagb";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "subvertpy" = super.buildPythonPackage {
     name = "subvertpy-0.10.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/76/99fa82affce75f5ac0f7dbe513796c3f37311ace0c68e1b063683b4f9b99/subvertpy-0.10.1.tar.gz";
       sha256 = "061ncy9wjz3zyv527avcrdyk0xygyssyy7p1644nhzhwp8zpybij";
     };
     meta = {
       license = [ pkgs.lib.licenses.lgpl21Plus pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "termcolor" = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "traitlets" = super.buildPythonPackage {
     name = "traitlets-4.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."six"
       self."decorator"
       self."enum34"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/75/b0/43deb021bc943f18f07cbe3dac1d681626a48997b7ffa1e7fb14ef922b21/traitlets-4.3.3.tar.gz";
       sha256 = "1xsrwgivpkxlbr4dfndfsi098s29yqgswgjc1qqn69yxklvfw8yh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "translationstring" = super.buildPythonPackage {
     name = "translationstring-1.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "venusian" = super.buildPythonPackage {
     name = "venusian-1.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/6f/40a9d43ac77cb51cb62be5b5662d170f43f8037bdc4eab56336c4ca92bb7/venusian-1.2.0.tar.gz";
       sha256 = "0ghyx66g8ikx9nx1mnwqvdcqm11i1vlq0hnvwl50s48bp22q5v34";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "waitress" = super.buildPythonPackage {
     name = "waitress-1.3.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a6/e6/708da7bba65898e5d759ade8391b1077e49d07be0b0223c39f5be04def56/waitress-1.3.1.tar.gz";
       sha256 = "1iysl8ka3l4cdrr0r19fh1cv28q41mwpvgsb81ji7k4shkb0k3i7";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "wcwidth" = super.buildPythonPackage {
-    name = "wcwidth-0.1.7";
+    name = "wcwidth-0.1.9";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
-      sha256 = "0pn6dflzm609m4r3i8ik5ni9ijjbb5fa3vg1n7hn6vkd49r77wrx";
+      url = "https://files.pythonhosted.org/packages/25/9d/0acbed6e4a4be4fc99148f275488580968f44ddb5e69b8ceb53fc9df55a0/wcwidth-0.1.9.tar.gz";
+      sha256 = "1wf5ycjx8s066rdvr0fgz4xds9a8zhs91c4jzxvvymm1c8l8cwzf";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webob" = super.buildPythonPackage {
     name = "webob-1.8.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/1a/0c89c070ee2829c934cb6c7082287c822e28236a4fcf90063e6be7c35532/WebOb-1.8.5.tar.gz";
       sha256 = "11khpzaxc88q31v25ic330gsf56fwmbdc9b30br8mvp0fmwspah5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webtest" = super.buildPythonPackage {
-    name = "webtest-2.0.33";
+    name = "webtest-2.0.34";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webob"
       self."waitress"
       self."beautifulsoup4"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/a8/b0/ffc9413b637dbe26e291429bb0f6ed731e518d0cd03da28524a8fe2e8a8f/WebTest-2.0.33.tar.gz";
-      sha256 = "1l3z0cwqslsf4rcrhi2gr8kdfh74wn2dw76376i4g9i38gz8wd21";
+      url = "https://files.pythonhosted.org/packages/2c/74/a0e63feee438735d628631e2b70d82280276a930637ac535479e5fad9427/WebTest-2.0.34.tar.gz";
+      sha256 = "0x1y2c8z4fmpsny4hbp6ka37si2g10r5r2jwxhvv5mx7g3blq4bi";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "zipp" = super.buildPythonPackage {
-    name = "zipp-0.6.0";
+    name = "zipp-1.2.0";
     doCheck = false;
     propagatedBuildInputs = [
-      self."more-itertools"
+      self."contextlib2"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/57/dd/585d728479d97d25aeeb9aa470d36a4ad8d0ba5610f84e14770128ce6ff7/zipp-0.6.0.tar.gz";
-      sha256 = "13ndkf7vklw978a4gdl1yfvn8hch28429a0iam67sg4nrp5v261p";
+      url = "https://files.pythonhosted.org/packages/78/08/d52f0ea643bc1068d6dc98b412f4966a9b63255d20911a23ac3220c033c4/zipp-1.2.0.tar.gz";
+      sha256 = "1c91lnv1bxjimh8as27hz7bghsjkkbxn1d37xq7in9c82iai0167";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "zope.deprecation" = super.buildPythonPackage {
     name = "zope.deprecation-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/34/da/46e92d32d545dd067b9436279d84c339e8b16de2ca393d7b892bc1e1e9fd/zope.deprecation-4.4.0.tar.gz";
       sha256 = "1pz2cv7gv9y1r3m0bdv7ks1alagmrn5msm5spwdzkb2by0w36i8d";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.interface" = super.buildPythonPackage {
     name = "zope.interface-4.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4e/d0/c9d16bd5b38de44a20c6dc5d5ed80a49626fafcb3db9f9efdc2a19026db6/zope.interface-4.6.0.tar.gz";
       sha256 = "1rgh2x3rcl9r0v0499kf78xy86rnmanajf4ywmqb943wpk50sg8v";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
 
 ### Test requirements
 
   
 }

requirements.txt

@@ -1,48 +1,48 b''
 ## dependencies
 
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
 
 dogpile.cache==0.9.0
 dogpile.core==0.4.1
 decorator==4.1.2
 dulwich==0.13.0
 hgsubversion==1.9.3
 hg-evolve==9.1.0
 mako==1.1.0
 markupsafe==1.1.1
 mercurial==5.1.1
 msgpack-python==0.5.6
 
-pastedeploy==2.0.1
+pastedeploy==2.1.0
 pyramid==1.10.4
 pyramid-mako==1.1.0
 pygit2==0.28.2
 
 repoze.lru==0.7
-redis==3.3.11
+redis==3.4.1
 simplejson==3.16.0
 subprocess32==3.5.4
 subvertpy==0.10.1
 
 six==1.11.0
 translationstring==1.3
 webob==1.8.5
 zope.deprecation==4.4.0
 zope.interface==4.6.0
 
 ## http servers
-gevent==1.4.0
+gevent==1.5.0
 greenlet==0.4.15
 gunicorn==19.9.0
 waitress==1.3.1
 
 ## debug
-ipdb==0.12.0
+ipdb==0.13.2
 ipython==5.1.0
 
 ## test related requirements
 -r requirements_test.txt
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_pinned.txt

@@ -1,18 +1,18 b''
 # contains not directly required libraries we want to pin the version.
 
 atomicwrites==1.3.0
 attrs==19.3.0
 contextlib2==0.6.0.post1
 cffi==1.12.3
-hupper==1.9.1
-importlib-metadata==0.23
-packaging==19.2.0
+hupper==1.10.2
+importlib-metadata==1.6.0
+packaging==20.3
 pathlib2==2.3.5
 pygments==2.4.2
-pyparsing==2.4.5
-psutil==5.6.5
+pyparsing==2.4.7
+psutil==5.7.0
 pluggy==0.13.1
 scandir==1.10.0
 setproctitle==1.1.10
 venusian==1.2.0
-wcwidth==0.1.7
+wcwidth==0.1.9

requirements_test.txt

@@ -1,16 +1,16 b''
 # test related requirements
 pytest==4.6.5
 py==1.8.0
 pytest-cov==2.7.1
 pytest-sugar==0.9.2
 pytest-runner==5.1.0
 pytest-profiling==1.7.0
 pytest-timeout==1.3.3
 gprof2dot==2017.9.19
 
 mock==3.0.5
 cov-core==1.15.0
 coverage==4.5.4
 
-webtest==2.0.33
+webtest==2.0.34
 beautifulsoup4==4.6.3

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.18.3
  No newline at end of file
+4.19.0
  No newline at end of file

vcsserver/__init__.py

@@ -1,28 +1,28 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import pkgutil
 
 
 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
 
 # link to config for pyramid
 CONFIG = {}
 
 # Populated with the settings dictionary from application init in
 #
 PYRAMID_SETTINGS = {}

vcsserver/base.py

@@ -1,76 +1,76 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import sys
 import traceback
 import logging
 import urlparse
 
 from vcsserver.lib.rc_cache import region_meta
 log = logging.getLogger(__name__)
 
 
 class RepoFactory(object):
     """
     Utility to create instances of repository
 
     It provides internal caching of the `repo` object based on
     the :term:`call context`.
     """
     repo_type = None
 
     def __init__(self):
         self._cache_region = region_meta.dogpile_cache_regions['repo_object']
 
     def _create_config(self, path, config):
         config = {}
         return config
 
     def _create_repo(self, wire, create):
         raise NotImplementedError()
 
     def repo(self, wire, create=False):
         raise NotImplementedError()
 
 
 def obfuscate_qs(query_string):
     if query_string is None:
         return None
 
     parsed = []
     for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
         if k in ['auth_token', 'api_key']:
             v = "*****"
         parsed.append((k, v))
 
     return '&'.join('{}{}'.format(
         k, '={}'.format(v) if v else '') for k, v in parsed)
 
 
 def raise_from_original(new_type):
     """
     Raise a new exception type with original args and traceback.
     """
     exc_type, exc_value, exc_traceback = sys.exc_info()
     new_exc = new_type(*exc_value.args)
     # store the original traceback into the new exc
     new_exc._org_exc_tb = traceback.format_exc(exc_traceback)
 
     try:
         raise new_exc, None, exc_traceback
     finally:
         del exc_traceback

vcsserver/exceptions.py

@@ -1,117 +1,121 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """
 Special exception handling over the wire.
 
 Since we cannot assume that our client is able to import our exception classes,
 this module provides a "wrapping" mechanism to raise plain exceptions
 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
 different error conditions.
 """
 
 from pyramid.httpexceptions import HTTPLocked, HTTPForbidden
 
 
 def _make_exception(kind, org_exc, *args):
     """
     Prepares a base `Exception` instance to be sent over the wire.
 
     To give our caller a hint what this is about, it will attach an attribute
     `_vcs_kind` to the exception.
     """
     exc = Exception(*args)
     exc._vcs_kind = kind
     exc._org_exc = org_exc
     exc._org_exc_tb = getattr(org_exc, '_org_exc_tb', '')
     return exc
 
 
 def AbortException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('abort', org_exc, *args)
     return _make_exception_wrapper
 
 
 def ArchiveException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('archive', org_exc, *args)
     return _make_exception_wrapper
 
 
 def LookupException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('lookup', org_exc, *args)
     return _make_exception_wrapper
 
 
 def VcsException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('error', org_exc, *args)
     return _make_exception_wrapper
 
 
 def RepositoryLockedException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('repo_locked', org_exc, *args)
     return _make_exception_wrapper
 
 
 def RepositoryBranchProtectedException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('repo_branch_protected', org_exc, *args)
     return _make_exception_wrapper
 
 
 def RequirementException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('requirement', org_exc, *args)
     return _make_exception_wrapper
 
 
 def UnhandledException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('unhandled', org_exc, *args)
     return _make_exception_wrapper
 
 
 def URLError(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('url_error', org_exc, *args)
     return _make_exception_wrapper
 
 
 def SubrepoMergeException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('subrepo_merge_error', org_exc, *args)
     return _make_exception_wrapper
 
 
 class HTTPRepoLocked(HTTPLocked):
     """
     Subclass of HTTPLocked response that allows to set the title and status
     code via constructor arguments.
     """
     def __init__(self, title, status_code=None, **kwargs):
         self.code = status_code or HTTPLocked.code
         self.title = title
         super(HTTPRepoLocked, self).__init__(**kwargs)
 
 
 class HTTPRepoBranchProtected(HTTPForbidden):
     def __init__(self, *args, **kwargs):
         super(HTTPForbidden, self).__init__(*args, **kwargs)
+
+
+class RefNotFoundException(KeyError):
+    pass

vcsserver/git.py

@@ -1,1181 +1,1192 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import collections
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import traceback
 import urllib
 import urllib2
 from functools import wraps
 
 import more_itertools
 import pygit2
 from pygit2 import Repository as LibGit2Repo
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str, safe_int, safe_unicode
 from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
 from vcsserver.vcs_base import RemoteBase
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 PEELED_REF_MARKER = '^{}'
 
 
 log = logging.getLogger(__name__)
 
 
 def str_to_dulwich(value):
     """
     Dulwich 0.10.1a requires `unicode` objects to be passed in.
     """
     return value.decode(settings.WIRE_ENCODING)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
 
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
             exc = exceptions.LookupException(org_exc=e)
             raise exc(safe_str(e))
         except (HangupException, UnexpectedCommandError) as e:
             exc = exceptions.VcsException(org_exc=e)
             raise exc(safe_str(e))
         except Exception as e:
             # NOTE(marcink): becuase of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
                 #log.exception("Unhandled exception in git remote call")
                 #raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class Repository(LibGit2Repo):
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.free()
 
 
 class GitFactory(RepoFactory):
     repo_type = 'git'
 
     def _create_repo(self, wire, create, use_libgit2=False):
         if use_libgit2:
             return Repository(wire['path'])
         else:
             repo_path = str_to_dulwich(wire['path'])
             return Repo(repo_path)
 
     def repo(self, wire, create=False, use_libgit2=False):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create, use_libgit2)
 
     def repo_libgit2(self, wire):
         return self.repo(wire, use_libgit2=True)
 
 
 class GitRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
         self._bulk_methods = {
             "date": self.date,
             "author": self.author,
             "branch": self.branch,
             "message": self.message,
             "parents": self.parents,
             "_commit": self.revision,
         }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
             return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
         return {}
 
     def _remote_conf(self, config):
         params = [
             '-c', 'core.askpass=""',
         ]
         ssl_cert_dir = config.get('vcs_ssl_dir')
         if ssl_cert_dir:
             params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
         return params
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = 'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return stdout.strip()
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
 
             try:
                 has_head = repo.head.name
                 if has_head:
                     return False
 
                 # NOTE(marcink): check again using more expensive method
                 return repo.is_empty
             except Exception:
                 pass
 
             return True
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id):
             try:
                 repo_init = self._factory.repo_libgit2(wire)
                 with repo_init as repo:
                     pass
             except pygit2.GitError:
                 path = wire.get('path')
                 tb = traceback.format_exc()
                 log.debug("Invalid Git path `%s`, tb: %s", path, tb)
                 return False
 
             return True
         return _assert_correct_path(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             return repo.is_bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob_obj = repo[sha]
             blob = blob_obj.data
             return blob
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _blob_raw_length(_repo_id, _sha):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[sha]
                 return blob.size
 
         return _blob_raw_length(repo_id, sha)
 
     def _parse_lfs_pointer(self, raw_content):
 
         spec_string = 'version https://git-lfs.github.com/spec'
         if raw_content and raw_content.startswith(spec_string):
             pattern = re.compile(r"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
             ^size[ ](?P<oid_size>[0-9]+)\n
             (?:\n)?
             """, re.VERBOSE | re.MULTILINE)
             match = pattern.match(raw_content)
             if match:
                 return match.groupdict()
 
         return {}
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob = repo[commit_id]
                 if blob.is_binary:
                     return {}
 
                 return self._parse_lfs_pointer(blob.data)
 
         return _is_large_file(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _tree_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 blob_obj = repo[tree_id]
                 return blob_obj.is_binary
 
         return _is_binary(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
 
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.has_oid()
 
         return False
 
     @reraise_safe_exceptions
     def store_path(self, wire, oid):
         conf = self._wire_to_config(wire)
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
         raise ValueError('Unable to fetch oid with path {}'.format(oid))
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _rev, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
                     args = [wire, rev]
                     result[attr] = method(*args)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(
                         "Unknown bulk attribute: %s" % attr)
             return result
 
         return _bulk_request(repo_id, rev, sorted(pre_load))
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
     def _type_id_to_name(self, type_id):
         return {
             1: b'commit',
             2: b'tree',
             3: b'blob',
             4: b'tag'
         }[type_id]
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError()(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
         remote_refs = self.pull(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     @reraise_safe_exceptions
     def branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _branch(_context_uid, _repo_id, _commit_id):
             regex = re.compile('^refs/heads')
 
             def filter_with(ref):
                 return regex.match(ref[0]) and ref[1] == _commit_id
 
             branches = filter(filter_with, self.get_refs(wire).items())
             return [x[0].split('refs/heads/')[-1] for x in branches]
 
         return _branch(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def commit_branches(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _commit_branches(_context_uid, _repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 branches = [x for x in repo.branches.with_commit(_commit_id)]
                 return branches
 
         return _commit_branches(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             blob = objects.Blob()
             blob.set_raw_string(content)
             repo.object_store.add_object(blob)
             return blob.id
 
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
         commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
                 parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.iteritems():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
         self.create_branch(wire, branch, commit.id)
 
         # dulwich set-ref
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             for k in remote_refs:
                 if k.endswith(PEELED_REF_MARKER):
                     log.debug("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
                 repo['HEAD'] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_fetch(self, wire, url, refs=None, all_refs=False):
         repo = self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
 
         config = self._wire_to_config(wire)
         # get all remote refs we'll use to fetch later
         cmd = ['ls-remote']
         if not all_refs:
             cmd += ['--heads', '--tags']
         cmd += [url]
         output, __ = self.run_git_command(
             wire, cmd, fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         remote_refs = collections.OrderedDict()
         fetch_refs = []
 
         for ref_line in output.splitlines():
             sha, ref = ref_line.split('\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
                 continue
             if ref.endswith(PEELED_REF_MARKER):
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
             if ref in ['HEAD']:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
                 fetch_refs.append('{}:{}'.format(ref, ref))
             elif not refs:
                 fetch_refs.append('{}:{}'.format(ref, ref))
         log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
 
         if fetch_refs:
             for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
                 fetch_refs_chunks = list(chunk)
                 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
                 _out, _err = self.run_git_command(
                     wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
                     extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url, refs=None):
         if not self.check_url(url, wire):
             return
         config = self._wire_to_config(wire)
         self._factory.repo(wire)
         self.run_git_command(
             wire, ['push', url, '--mirror'], fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
-    def get_object(self, wire, sha):
+    def get_object(self, wire, sha, maybe_unreachable=False):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_object(_context_uid, _repo_id, _sha):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
 
                 missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
                 try:
                     commit = repo.revparse_single(sha)
-                except (KeyError, ValueError) as e:
+                except KeyError:
+                    # NOTE(marcink): KeyError doesn't give us any meaningful information
+                    # here, we instead give something more explicit
+                    e = exceptions.RefNotFoundException('SHA: %s not found', sha)
+                    raise exceptions.LookupException(e)(missing_commit_err)
+                except ValueError as e:
                     raise exceptions.LookupException(e)(missing_commit_err)
 
                 is_tag = False
                 if isinstance(commit, pygit2.Tag):
                     commit = repo.get(commit.target)
                     is_tag = True
 
                 check_dangling = True
                 if is_tag:
                     check_dangling = False
 
+                if check_dangling and maybe_unreachable:
+                    check_dangling = False
+
                 # we used a reference and it parsed means we're not having a dangling commit
                 if sha != commit.hex:
                     check_dangling = False
 
                 if check_dangling:
                     # check for dangling commit
                     for branch in repo.branches.with_commit(commit.hex):
                         if branch:
                             break
                     else:
-                        raise exceptions.LookupException(None)(missing_commit_err)
+                        # NOTE(marcink): Empty error doesn't give us any meaningful information
+                        # here, we instead give something more explicit
+                        e = exceptions.RefNotFoundException('SHA: %s not found in branches', sha)
+                        raise exceptions.LookupException(e)(missing_commit_err)
 
                 commit_id = commit.hex
                 type_id = commit.type
 
                 return {
                     'id': commit_id,
                     'type': self._type_id_to_name(type_id),
                     'commit_id': commit_id,
                     'idx': 0
                 }
 
         return _get_object(context_uid, repo_id, sha)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_refs(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 regex = re.compile('^refs/(heads|tags)/')
                 return {x.name: x.target.hex for x in
                         filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
 
         return _get_refs(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def get_branch_pointers(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_branch_pointers(_context_uid, _repo_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             regex = re.compile('^refs/heads')
             with repo_init as repo:
                 branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
                 return {x.target.hex: x.shorthand for x in branches}
 
         return _get_branch_pointers(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def head(self, wire, show_exc=True):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _head(_context_uid, _repo_id, _show_exc):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     return repo.head.peel().hex
                 except Exception:
                     if show_exc:
                         raise
         return _head(context_uid, repo_id, show_exc)
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _revision(_context_uid, _repo_id, _rev):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[rev]
                 obj_data = {
                     'id': commit.id.hex,
                 }
                 # tree objects itself don't have tree_id attribute
                 if hasattr(commit, 'tree_id'):
                     obj_data['tree'] = commit.tree_id.hex
 
                 return obj_data
         return _revision(context_uid, repo_id, rev)
 
     @reraise_safe_exceptions
     def date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _date(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'commit_time'):
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
                 else:
                     commit = commit.get_object()
                     commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
 
                 # TODO(marcink): check dulwich difference of offset vs timezone
                 return [commit_time, commit_time_offset]
         return _date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def author(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _author(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
 
                 if hasattr(commit, 'author'):
                     author = commit.author
                 else:
                     author = commit.get_object().author
 
                 if author.email:
                     return u"{} <{}>".format(author.name, author.email)
 
                 try:
                     return u"{}".format(author.name)
                 except Exception:
                     return u"{}".format(safe_unicode(author.raw_name))
 
         return _author(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def message(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _message(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 return commit.message
         return _message(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _parents(_repo_id, _commit_id):
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 commit = repo[commit_id]
                 if hasattr(commit, 'parent_ids'):
                     parent_ids = commit.parent_ids
                 else:
                     parent_ids = commit.get_object().parent_ids
 
                 return [x.hex for x in parent_ids]
         return _parents(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _children(_repo_id, _commit_id):
             output, __ = self.run_git_command(
                 wire, ['rev-list', '--all', '--children'])
 
             child_ids = []
             pat = re.compile(r'^%s' % commit_id)
             for l in output.splitlines():
                 if pat.match(l):
                     found_ids = l.split(' ')[1:]
                     child_ids.extend(found_ids)
 
             return child_ids
         return _children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.create(key, value, force=True)
 
     @reraise_safe_exceptions
     def create_branch(self, wire, branch_name, commit_id, force=False):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             commit = repo[commit_id]
 
             if force:
                 repo.branches.local.create(branch_name, commit, force=force)
             elif not repo.branches.get(branch_name):
                 # create only if that branch isn't existing
                 repo.branches.local.create(branch_name, commit, force=force)
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tag_remove(self, wire, tag_name):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             key = 'refs/tags/{}'.format(tag_name)
             repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
         # TODO(marcink): remove this seems it's only used by tests
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
     def tree_and_type_for_path(self, wire, commit_id, path):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
             repo_init = self._factory.repo_libgit2(wire)
 
             with repo_init as repo:
                 commit = repo[commit_id]
                 try:
                     tree = commit.tree[path]
                 except KeyError:
                     return None, None, None
 
                 return tree.id.hex, tree.type, tree.filemode
         return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _tree_items(_repo_id, _tree_id):
 
             repo_init = self._factory.repo_libgit2(wire)
             with repo_init as repo:
                 try:
                     tree = repo[tree_id]
                 except KeyError:
                     raise ObjectMissing('No tree with id: {}'.format(tree_id))
 
                 result = []
                 for item in tree:
                     item_sha = item.hex
                     item_mode = item.filemode
                     item_type = item.type
 
                     if item_type == 'commit':
                         # NOTE(marcink): submodules we translate to 'link' for backward compat
                         item_type = 'link'
 
                     result.append((item.name, item_mode, item_sha, item_type))
                 return result
         return _tree_items(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         """
         Old version that uses subprocess to call diff
         """
 
         flags = [
             '-U%s' % context, '--patch',
             '--binary',
             '--find-renames',
             '--no-indent-heuristic',
             # '--indent-heuristic',
             #'--full-index',
             #'--abbrev=40'
         ]
 
         if opt_ignorews:
             flags.append('--ignore-all-space')
 
         if commit_id_1 == self.EMPTY_COMMIT:
             cmd = ['show'] + flags + [commit_id_2]
         else:
             cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
 
         if file_filter:
             cmd.extend(['--', file_filter])
 
         diff, __ = self.run_git_command(wire, cmd)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if commit_id_1 == self.EMPTY_COMMIT:
             lines = diff.splitlines()
             x = 0
             for line in lines:
                 if line.startswith('diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
             diff = '\n'.join(lines[x:]) + '\n'
         return diff
 
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
         repo_init = self._factory.repo_libgit2(wire)
         with repo_init as repo:
             swap = True
             flags = 0
             flags |= pygit2.GIT_DIFF_SHOW_BINARY
 
             if opt_ignorews:
                 flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
 
             if commit_id_1 == self.EMPTY_COMMIT:
                 comm1 = repo[commit_id_2]
                 diff_obj = comm1.tree.diff_to_tree(
                     flags=flags, context_lines=context, swap=swap)
 
             else:
                 comm1 = repo[commit_id_2]
                 comm2 = repo[commit_id_1]
                 diff_obj = comm1.tree.diff_to_tree(
                     comm2.tree, flags=flags, context_lines=context, swap=swap)
             similar_flags = 0
             similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
             diff_obj.find_similar(flags=similar_flags)
 
             if file_filter:
                 for p in diff_obj:
                     if p.delta.old_file.path == file_filter:
                         return p.patch or ''
                 # fo matching path == no diff
                 return ''
             return diff_obj.patch or ''
 
     @reraise_safe_exceptions
     def node_history(self, wire, commit_id, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
             # optimize for n==1, rev-list is much faster for that use-case
             if limit == 1:
                 cmd = ['rev-list', '-1', commit_id, '--', path]
             else:
                 cmd = ['log']
                 if limit:
                     cmd.extend(['-n', str(safe_int(limit, 0))])
                 cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
 
             output, __ = self.run_git_command(wire, cmd)
             commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
 
             return [x for x in commit_ids]
         return _node_history(context_uid, repo_id, commit_id, path, limit)
 
     @reraise_safe_exceptions
     def node_annotate(self, wire, commit_id, path):
 
         cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for boundaries
         # -r commit_id ==> blames for the given commit
         output, __ = self.run_git_command(wire, cmd)
 
         result = []
         for i, blame_line in enumerate(output.split('\n')[:-1]):
             line_no = i + 1
             commit_id, line = re.split(r' ', blame_line, 1)
             result.append((line_no, commit_id, line))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id):
 
             cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
             try:
                 output, __ = self.run_git_command(wire, cmd)
                 return output.splitlines()
             except Exception:
                 # Can be raised for empty repositories
                 return []
         return _get_all_commit_ids(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         if '_copts' in opts:
             _copts.extend(opts['_copts'] or [])
             del opts['_copts']
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
         _opts = {'env': gitenv, 'shell': False}
 
         proc = None
         try:
             _opts.update(opts)
             proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(proc), ''.join(proc.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
         finally:
             if proc:
                 proc.close()
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_git_hooks
         bare = self.bare(wire)
         path = wire['path']
         return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_git_pre_hook_version, get_git_post_hook_version)
         bare = self.bare(wire)
         path = wire['path']
         return {
             'pre_version': get_git_pre_hook_version(path, bare),
             'post_version': get_git_post_hook_version(path, bare),
         }

vcsserver/git_lfs/__init__.py

@@ -1,19 +1,19 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 from app import create_app

vcsserver/git_lfs/app.py

@@ -1,292 +1,292 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import re
 import logging
 from wsgiref.util import FileWrapper
 
 import simplejson as json
 from pyramid.config import Configurator
 from pyramid.response import Response, FileIter
 from pyramid.httpexceptions import (
     HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
     HTTPUnprocessableEntity)
 
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
 from vcsserver.utils import safe_int
 
 log = logging.getLogger(__name__)
 
 
 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def write_response_error(http_exception, text=None):
     content_type = GIT_LFS_CONTENT_TYPE + '+json'
     _exception = http_exception(content_type=content_type)
     _exception.content_type = content_type
     if text:
         _exception.body = json.dumps({'message': text})
     log.debug('LFS: writing response of type %s to client with text:%s',
               http_exception, text)
     return _exception
 
 
 class AuthHeaderRequired(object):
     """
     Decorator to check if request has proper auth-header
     """
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         request = fargs[1]
         auth = request.authorization
         if not auth:
             return write_response_error(HTTPForbidden)
         return func(*fargs[1:], **fkwargs)
 
 
 # views
 
 def lfs_objects(request):
     # indicate not supported, V1 API
     log.warning('LFS: v1 api not supported, reporting it back to client')
     return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
 
 
 @AuthHeaderRequired()
 def lfs_objects_batch(request):
     """
     The client sends the following information to the Batch endpoint to transfer some objects:
 
         operation - Should be download or upload.
         transfers - An optional Array of String identifiers for transfer
             adapters that the client has configured. If omitted, the basic
             transfer adapter MUST be assumed by the server.
         objects - An Array of objects to download.
         oid - String OID of the LFS object.
         size - Integer byte size of the LFS object. Must be at least zero.
     """
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     auth = request.authorization
     repo = request.matchdict.get('repo')
     data = request.json
     operation = data.get('operation')
     http_scheme = request.registry.git_lfs_http_scheme
 
     if operation not in ('download', 'upload'):
         log.debug('LFS: unsupported operation:%s', operation)
         return write_response_error(
             HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
 
     if 'objects' not in data:
         log.debug('LFS: missing objects data')
         return write_response_error(
             HTTPBadRequest, 'missing objects data')
 
     log.debug('LFS: handling operation of type: %s', operation)
 
     objects = []
     for o in data['objects']:
         try:
             oid = o['oid']
             obj_size = o['size']
         except KeyError:
             log.exception('LFS, failed to extract data')
             return write_response_error(
                 HTTPBadRequest, 'unsupported data in objects')
 
         obj_data = {'oid': oid}
 
         obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid, 
                                      _scheme=http_scheme)
         obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
                                             _scheme=http_scheme)
         store = LFSOidStore(
             oid, repo, store_location=request.registry.git_lfs_store_path)
         handler = OidHandler(
             store, repo, auth, oid, obj_size, obj_data,
             obj_href, obj_verify_href)
 
         # this verifies also OIDs
         actions, errors = handler.exec_operation(operation)
         if errors:
             log.warning('LFS: got following errors: %s', errors)
             obj_data['errors'] = errors
 
         if actions:
             obj_data['actions'] = actions
 
         obj_data['size'] = obj_size
         obj_data['authenticated'] = True
         objects.append(obj_data)
 
     result = {'objects': objects, 'transfer': 'basic'}
     log.debug('LFS Response %s', safe_result(result))
 
     return result
 
 
 def lfs_objects_oid_upload(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     engine = store.get_engine(mode='wb')
     log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
 
     body = request.environ['wsgi.input']
 
     with engine as f:
         blksize = 64 * 1024  # 64kb
         while True:
             # read in chunks as stream comes in from Gunicorn
             # this is a specific Gunicorn support function.
             # might work differently on waitress
             chunk = body.read(blksize)
             if not chunk:
                 break
             f.write(chunk)
 
     return {'upload': 'ok'}
 
 
 def lfs_objects_oid_download(request):
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
 
     # TODO(marcink): support range header ?
     # Range: bytes=0-, `bytes=(\d+)\-.*`
 
     f = open(store.oid_path, 'rb')
     response = Response(
         content_type='application/octet-stream', app_iter=FileIter(f))
     response.headers.add('X-RC-LFS-Response-Oid', str(oid))
     return response
 
 
 def lfs_objects_verify(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
 
     data = request.json
     oid = data.get('oid')
     size = safe_int(data.get('size'))
 
     if not (oid and size):
         return write_response_error(
             HTTPBadRequest, 'missing oid and size in request data')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, 'oid `%s` does not exists in store' % oid)
 
     store_size = store.size_oid()
     if store_size != size:
         msg = 'requested file size mismatch store size:%s requested:%s' % (
             store_size, size)
         return write_response_error(
             HTTPUnprocessableEntity, msg)
 
     return {'message': {'size': 'ok', 'in_store': 'ok'}}
 
 
 def lfs_objects_lock(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS locking api not supported')
 
 
 def not_found(request):
     return write_response_error(
         HTTPNotFound, 'request path not found')
 
 
 def lfs_disabled(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS disabled for this repo')
 
 
 def git_lfs_app(config):
 
     # v1 API deprecation endpoint
     config.add_route('lfs_objects',
                      '/{repo:.*?[^/]}/info/lfs/objects')
     config.add_view(lfs_objects, route_name='lfs_objects',
                     request_method='POST', renderer='json')
 
     # locking API
     config.add_route('lfs_objects_lock',
                      '/{repo:.*?[^/]}/info/lfs/locks')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
                     request_method=('POST', 'GET'), renderer='json')
 
     config.add_route('lfs_objects_lock_verify',
                      '/{repo:.*?[^/]}/info/lfs/locks/verify')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
                     request_method=('POST', 'GET'), renderer='json')
 
     # batch API
     config.add_route('lfs_objects_batch',
                      '/{repo:.*?[^/]}/info/lfs/objects/batch')
     config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
                     request_method='POST', renderer='json')
 
     # oid upload/download API
     config.add_route('lfs_objects_oid',
                      '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
     config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
                     request_method='PUT', renderer='json')
     config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
                     request_method='GET', renderer='json')
 
     # verification API
     config.add_route('lfs_objects_verify',
                      '/{repo:.*?[^/]}/info/lfs/verify')
     config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
                     request_method='POST', renderer='json')
 
     # not found handler for API
     config.add_notfound_view(not_found, renderer='json')
 
 
 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
     config = Configurator()
     if git_lfs_enabled:
         config.include(git_lfs_app)
         config.registry.git_lfs_store_path = git_lfs_store_path
         config.registry.git_lfs_http_scheme = git_lfs_http_scheme
     else:
         # not found handler for API, reporting disabled LFS support
         config.add_notfound_view(lfs_disabled, renderer='json')
 
     app = config.make_wsgi_app()
     return app

vcsserver/git_lfs/lib.py

@@ -1,175 +1,175 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import shutil
 import logging
 from collections import OrderedDict
 
 log = logging.getLogger(__name__)
 
 
 class OidHandler(object):
 
     def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
                  obj_verify_href=None):
         self.current_store = store
         self.repo_name = repo_name
         self.auth = auth
         self.oid = oid
         self.obj_size = obj_size
         self.obj_data = obj_data
         self.obj_href = obj_href
         self.obj_verify_href = obj_verify_href
 
     def get_store(self, mode=None):
         return self.current_store
 
     def get_auth(self):
         """returns auth header for re-use in upload/download"""
         return " ".join(self.auth)
 
     def download(self):
 
         store = self.get_store()
         response = None
         has_errors = None
 
         if not store.has_oid():
             # error reply back to client that something is wrong with dl
             err_msg = 'object: {} does not exist in store'.format(store.oid)
             has_errors = OrderedDict(
                 error=OrderedDict(
                     code=404,
                     message=err_msg
                 )
             )
 
         download_action = OrderedDict(
             href=self.obj_href,
             header=OrderedDict([("Authorization", self.get_auth())])
         )
         if not has_errors:
             response = OrderedDict(download=download_action)
         return response, has_errors
 
     def upload(self, skip_existing=True):
         """
         Write upload action for git-lfs server
         """
 
         store = self.get_store()
         response = None
         has_errors = None
 
         # verify if we have the OID before, if we do, reply with empty
         if store.has_oid():
             log.debug('LFS: store already has oid %s', store.oid)
 
             # validate size
             store_size = store.size_oid()
             size_match = store_size == self.obj_size
             if not size_match:
                 log.warning(
                     'LFS: size mismatch for oid:%s, in store:%s expected: %s',
                     self.oid, store_size, self.obj_size)
             elif skip_existing:
                 log.debug('LFS: skipping further action as oid is existing')
                 return response, has_errors
 
         chunked = ("Transfer-Encoding", "chunked")
         upload_action = OrderedDict(
             href=self.obj_href,
             header=OrderedDict([("Authorization", self.get_auth()), chunked])
         )
         if not has_errors:
             response = OrderedDict(upload=upload_action)
             # if specified in handler, return the verification endpoint
             if self.obj_verify_href:
                 verify_action = OrderedDict(
                     href=self.obj_verify_href,
                     header=OrderedDict([("Authorization", self.get_auth())])
                 )
                 response['verify'] = verify_action
         return response, has_errors
 
     def exec_operation(self, operation, *args, **kwargs):
         handler = getattr(self, operation)
         log.debug('LFS: handling request using %s handler', handler)
         return handler(*args, **kwargs)
 
 
 class LFSOidStore(object):
 
     def __init__(self, oid, repo, store_location=None):
         self.oid = oid
         self.repo = repo
         self.store_path = store_location or self.get_default_store()
         self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
         self.oid_path = os.path.join(self.store_path, oid)
         self.fd = None
 
     def get_engine(self, mode):
         """
         engine = .get_engine(mode='wb')
         with engine as f:
             f.write('...')
         """
 
         class StoreEngine(object):
             def __init__(self, mode, store_path, oid_path, tmp_oid_path):
                 self.mode = mode
                 self.store_path = store_path
                 self.oid_path = oid_path
                 self.tmp_oid_path = tmp_oid_path
 
             def __enter__(self):
                 if not os.path.isdir(self.store_path):
                     os.makedirs(self.store_path)
 
                 # TODO(marcink): maybe write metadata here with size/oid ?
                 fd = open(self.tmp_oid_path, self.mode)
                 self.fd = fd
                 return fd
 
             def __exit__(self, exc_type, exc_value, traceback):
                 # close tmp file, and rename to final destination
                 self.fd.close()
                 shutil.move(self.tmp_oid_path, self.oid_path)
 
         return StoreEngine(
             mode, self.store_path, self.oid_path, self.tmp_oid_path)
 
     def get_default_store(self):
         """
         Default store, consistent with defaults of Mercurial large files store
         which is /home/username/.cache/largefiles
         """
         user_home = os.path.expanduser("~")
         return os.path.join(user_home, '.cache', 'lfs-store')
 
     def has_oid(self):
         return os.path.exists(os.path.join(self.store_path, self.oid))
 
     def size_oid(self):
         size = -1
 
         if self.has_oid():
             oid = os.path.join(self.store_path, self.oid)
             size = os.stat(oid).st_size
 
         return size

vcsserver/git_lfs/tests/__init__.py

@@ -1,16 +1,16 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -1,272 +1,272 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import pytest
 from webtest.app import TestApp as WebObTestApp
 import simplejson as json
 
 from vcsserver.git_lfs.app import create_app
 
 
 @pytest.fixture(scope='function')
 def git_lfs_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='http'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture(scope='function')
 def git_lfs_https_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='https'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture()
 def http_auth():
     return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
 
 
 class TestLFSApplication(object):
 
     def test_app_wrong_path(self, git_lfs_app):
         git_lfs_app.get('/repo/info/lfs/xxx', status=404)
 
     def test_app_deprecated_endpoint(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {u'message': u'LFS: v1 api not supported'}
 
     def test_app_lock_verify_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             u'message': u'GIT LFS locking api not supported'}
 
     def test_app_lock_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             u'message': u'GIT LFS locking api not supported'}
 
     def test_app_batch_api_missing_auth(self, git_lfs_app):
         git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=403)
 
     def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             u'message': u'unsupported operation mode: `None`'}
 
     def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={'operation': 'download'},
             status=400, extra_environ=http_auth)
         assert json.loads(response.text) == {
             u'message': u'missing objects data'}
 
     def test_app_batch_api_unsupported_data_in_objects(
             self, git_lfs_app, http_auth):
         params = {'operation': 'download',
                   'objects': [{}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             u'message': u'unsupported data in objects'}
 
     def test_app_batch_api_download_missing_object(
             self, git_lfs_app, http_auth):
         params = {'operation': 'download',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
             {u'authenticated': True,
              u'errors': {u'error': {
                  u'code': 404,
                  u'message': u'object: 123 does not exist in store'}},
              u'oid': u'123',
              u'size': u'1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_download(self, git_lfs_app, http_auth):
         oid = '456'
         oid_path = os.path.join(git_lfs_app._store, oid)
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write('OID_CONTENT')
 
         params = {'operation': 'download',
                   'objects': [{'oid': oid, 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
             {u'authenticated': True,
              u'actions': {
                  u'download': {
                      u'header': {u'Authorization': u'Basic XXXXX'},
                      u'href': u'http://localhost/repo/info/lfs/objects/456'},
              },
              u'oid': u'456',
              u'size': u'1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_upload(self, git_lfs_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
             {u'authenticated': True,
              u'actions': {
                  u'upload': {
                      u'header': {u'Authorization': u'Basic XXXXX',
                                  u'Transfer-Encoding': u'chunked'},
                      u'href': u'http://localhost/repo/info/lfs/objects/123'},
                  u'verify': {
                      u'header': {u'Authorization': u'Basic XXXXX'},
                      u'href': u'http://localhost/repo/info/lfs/verify'}
              },
              u'oid': u'123',
              u'size': u'1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_https_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
             {u'authenticated': True,
              u'actions': {
                  u'upload': {
                      u'header': {u'Authorization': u'Basic XXXXX',
                                  u'Transfer-Encoding': u'chunked'},
                      u'href': u'https://localhost/repo/info/lfs/objects/123'},
                  u'verify': {
                      u'header': {u'Authorization': u'Basic XXXXX'},
                      u'href': u'https://localhost/repo/info/lfs/verify'}
              },
              u'oid': u'123',
              u'size': u'1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_verify_api_missing_data(self, git_lfs_app):
         params = {'oid': 'missing'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=400)
 
         assert json.loads(response.text) == {
             u'message': u'missing oid and size in request data'}
 
     def test_app_verify_api_missing_obj(self, git_lfs_app):
         params = {'oid': 'missing', 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=404)
 
         assert json.loads(response.text) == {
             u'message': u'oid `missing` does not exists in store'}
 
     def test_app_verify_api_size_mismatch(self, git_lfs_app):
         oid = 'existing'
         oid_path = os.path.join(git_lfs_app._store, oid)
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write('OID_CONTENT')
 
         params = {'oid': oid, 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params, status=422)
 
         assert json.loads(response.text) == {
             u'message': u'requested file size mismatch '
                         u'store size:11 requested:1024'}
 
     def test_app_verify_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = os.path.join(git_lfs_app._store, oid)
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write('OID_CONTENT')
 
         params = {'oid': oid, 'size': 11}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params)
 
         assert json.loads(response.text) == {
             u'message': {u'size': u'ok', u'in_store': u'ok'}}
 
     def test_app_download_api_oid_not_existing(self, git_lfs_app):
         oid = 'missing'
 
         response = git_lfs_app.get(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
 
         assert json.loads(response.text) == {
             u'message': u'requested file with oid `missing` not found in store'}
 
     def test_app_download_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = os.path.join(git_lfs_app._store, oid)
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write('OID_CONTENT')
 
         response = git_lfs_app.get(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid))
         assert response
 
     def test_app_upload(self, git_lfs_app):
         oid = 'uploaded'
 
         response = git_lfs_app.put(
             '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
 
         assert json.loads(response.text) == {u'upload': u'ok'}
 
         # verify that we actually wrote that OID
         oid_path = os.path.join(git_lfs_app._store, oid)
         assert os.path.isfile(oid_path)
         assert 'CONTENT' == open(oid_path).read()

vcsserver/git_lfs/tests/test_lib.py

@@ -1,141 +1,141 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import pytest
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 
 
 @pytest.fixture()
 def lfs_store(tmpdir):
     repo = 'test'
     oid = '123456789'
     store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
     return store
 
 
 @pytest.fixture()
 def oid_handler(lfs_store):
     store = lfs_store
     repo = store.repo
     oid = store.oid
 
     oid_handler = OidHandler(
         store=store, repo_name=repo, auth=('basic', 'xxxx'),
         oid=oid,
         obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
         obj_verify_href='http://localhost/verify')
     return oid_handler
 
 
 class TestOidHandler(object):
 
     @pytest.mark.parametrize('exec_action', [
         'download',
         'upload',
     ])
     def test_exec_action(self, exec_action, oid_handler):
         handler = oid_handler.exec_operation(exec_action)
         assert handler
 
     def test_exec_action_undefined(self, oid_handler):
         with pytest.raises(AttributeError):
             oid_handler.exec_operation('wrong')
 
     def test_download_oid_not_existing(self, oid_handler):
         response, has_errors = oid_handler.exec_operation('download')
 
         assert response is None
         assert has_errors['error'] == {
             'code': 404,
             'message': 'object: 123456789 does not exist in store'}
 
     def test_download_oid(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write('CONTENT')
 
         response, has_errors = oid_handler.exec_operation('download')
 
         assert has_errors is None
         assert response['download'] == {
             'header': {'Authorization': 'basic xxxx'},
             'href': 'http://localhost/handle_oid'
         }
 
     def test_upload_oid_that_exists(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write('CONTENT')
         oid_handler.obj_size = 7
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response is None
 
     def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write('CONTENT')
 
         oid_handler.obj_size = 10240
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response['upload'] == {
             'header': {'Authorization': 'basic xxxx',
                        'Transfer-Encoding': 'chunked'},
             'href': 'http://localhost/handle_oid',
         }
 
     def test_upload_oid(self, oid_handler):
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response['upload'] == {
             'header': {'Authorization': 'basic xxxx',
                        'Transfer-Encoding': 'chunked'},
             'href': 'http://localhost/handle_oid'
         }
 
 
 class TestLFSStore(object):
     def test_write_oid(self, lfs_store):
         oid_location = lfs_store.oid_path
 
         assert not os.path.isfile(oid_location)
 
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
             f.write('CONTENT')
 
         assert os.path.isfile(oid_location)
 
     def test_detect_has_oid(self, lfs_store):
 
         assert lfs_store.has_oid() is False
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
             f.write('CONTENT')
 
         assert lfs_store.has_oid() is True
  No newline at end of file

vcsserver/git_lfs/utils.py

@@ -1,50 +1,50 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 import copy
 from functools import wraps
 
 
 def get_cython_compat_decorator(wrapper, func):
     """
     Creates a cython compatible decorator. The previously used
     decorator.decorator() function seems to be incompatible with cython.
 
     :param wrapper: __wrapper method of the decorator class
     :param func: decorated function
     """
     @wraps(func)
     def local_wrapper(*args, **kwds):
         return wrapper(func, *args, **kwds)
     local_wrapper.__wrapped__ = func
     return local_wrapper
 
 
 def safe_result(result):
     """clean result for better representation in logs"""
     clean_copy = copy.deepcopy(result)
 
     try:
         if 'objects' in clean_copy:
             for oid_data in clean_copy['objects']:
                 if 'actions' in oid_data:
                     for action_name, data in oid_data['actions'].items():
                         if 'header' in data:
                             data['header'] = {'Authorization': '*****'}
     except Exception:
         return result
 
     return clean_copy

vcsserver/hg.py

@@ -1,1009 +1,1009 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import logging
 import stat
 import urllib
 import urllib2
 import traceback
 
 from hgext import largefiles, rebase, purge
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
 from mercurial import verify
 from mercurial import repair
 
 import vcsserver
 from vcsserver import exceptions
 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError)
 from vcsserver.vcs_base import RemoteBase
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
 
     class LoggingUI(ui.ui):
         def status(self, *msg, **opts):
             log.info(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).status(*msg, **opts)
 
         def warn(self, *msg, **opts):
             log.warn(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).warn(*msg, **opts)
 
         def error(self, *msg, **opts):
             log.error(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).error(*msg, **opts)
 
         def note(self, *msg, **opts):
             log.info(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).note(*msg, **opts)
 
         def debug(self, *msg, **opts):
             log.debug(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).debug(*msg, **opts)
 
     baseui = LoggingUI()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     baseui.setconfig('ui', 'paginate', 'never')
     # for better Error reporting of Mercurial
     baseui.setconfig('ui', 'message-output', 'stderr')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig('worker', 'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig('extensions', 'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig('extensions', 'largefiles', '!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
 
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired) as e:
             raise_from_original(exceptions.AbortException(e))
         except RepoLookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except RequirementError as e:
             raise_from_original(exceptions.RequirementException(e))
         except RepoError as e:
             raise_from_original(exceptions.VcsException(e))
         except LookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException(e))
 
             raise
     return wrapper
 
 
 class MercurialFactory(RepoFactory):
     repo_type = 'hg'
 
     def _create_config(self, config, hooks=True):
         if not hooks:
             hooks_to_clean = frozenset((
                 'changegroup.repo_size', 'preoutgoing.pre_pull',
                 'outgoing.pull_logger', 'prechangegroup.pre_push'))
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         return instance(baseui, wire["path"], create)
 
     def repo(self, wire, create=False):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create)
 
 
 def patch_ui_message_output(baseui):
     baseui.setconfig('ui', 'quiet', 'false')
     output = io.BytesIO()
 
     def write(data, **unused_kwargs):
         output.write(data)
 
     baseui.status = write
     baseui.write = write
     baseui.warn = write
     baseui.debug = write
 
     return baseui, output
 
 
 class HgRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "obsolete": self.ctx_obsolete,
             "phase": self.ctx_phase,
             "hidden": self.ctx_hidden,
             "_file_paths": self.ctx_list,
         }
 
     def _get_ctx(self, repo, ref):
         return get_ctx(repo, ref)
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return util.version()
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo = self._factory.repo(wire)
 
         try:
             return len(repo) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
     @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
         elif kind == "tbz2":
             archiver = archival.tarit(archive_path, mtime, "bz2")
         elif kind == 'zip':
             archiver = archival.zipit(archive_path, mtime)
         else:
             raise exceptions.ArchiveException()(
                 'Remote does not support: "%s".' % kind)
 
         for f_path, f_mode, f_is_link, f_content in file_info:
             archiver.addfile(f_path, f_mode, f_is_link, f_content)
         archiver.done()
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _bookmarks(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             return dict(repo._bookmarks)
 
         return _bookmarks(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _branches(_context_uid, _repo_id, _normal, _closed):
             repo = self._factory.repo(wire)
             iter_branches = repo.branchmap().iterbranches()
             bt = {}
             for branch_name, _heads, tip, is_closed in iter_branches:
                 if normal and not is_closed:
                     bt[branch_name] = tip
                 if closed and is_closed:
                     bt[branch_name] = tip
 
             return bt
 
         return _branches(context_uid, repo_id, normal, closed)
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, commit_id, pre_load):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _bulk_request(_repo_id, _commit_id, _pre_load):
             result = {}
             for attr in pre_load:
                 try:
                     method = self._bulk_methods[attr]
                     result[attr] = method(wire, commit_id)
                 except KeyError as e:
                     raise exceptions.VcsException(e)(
                         'Unknown bulk attribute: "%s"' % attr)
             return result
 
         return _bulk_request(repo_id, commit_id, sorted(pre_load))
 
     @reraise_safe_exceptions
     def ctx_branch(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_branch(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.branch()
         return _ctx_branch(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_date(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_date(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.date()
         return _ctx_date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.description()
 
     @reraise_safe_exceptions
     def ctx_files(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_files(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.files()
 
         return _ctx_files(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = self._get_ctx(repo, revision)
         return list(ctx)
 
     @reraise_safe_exceptions
     def ctx_parents(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_parents(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return [parent.hex() for parent in ctx.parents()
                     if not (parent.hidden() or parent.obsolete())]
 
         return _ctx_parents(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_children(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_children(_repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return [child.hex() for child in ctx.children()
                     if not (child.hidden() or child.obsolete())]
 
         return _ctx_children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_phase(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_phase(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             # public=0, draft=1, secret=3
             return ctx.phase()
         return _ctx_phase(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_obsolete(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.obsolete()
         return _ctx_obsolete(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_hidden(self, wire, commit_id):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _ctx_hidden(_context_uid, _repo_id, _commit_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             return ctx.hidden()
         return _ctx_hidden(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
 
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
                     "Verifying if URL is a Mercurial repository: %s",
                     cleaned_uri)
                 ui = make_ui_from_config(config)
                 peer_checker = makepeer(ui, url)
                 peer_checker.lookup('tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
                         cleaned_uri)
             raise exceptions.URLError(e)(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
         log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
     def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             match_filter = match(file_filter[0], '', [file_filter[1]])
         else:
             match_filter = file_filter
-        opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
+        opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context, showfunc=1)
 
         try:
             return "".join(patch.diff(
                 repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts))
         except RepoLookupError as e:
             raise exceptions.LookupException(e)()
 
     @reraise_safe_exceptions
     def node_history(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
             repo = self._factory.repo(wire)
 
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(path)
 
             def history_iter():
                 limit_rev = fctx.rev()
                 for obj in reversed(list(fctx.filelog())):
                     obj = fctx.filectx(obj)
                     ctx = obj.changectx()
                     if ctx.hidden() or ctx.obsolete():
                         continue
 
                     if limit_rev >= obj.rev():
                         yield obj
 
             history = []
             for cnt, obj in enumerate(history_iter()):
                 if limit and cnt >= limit:
                     break
                 history.append(hex(obj.node()))
 
             return [x for x in history]
         return _node_history(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
     def node_history_untill(self, wire, revision, path, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _node_history_until(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(path)
 
             file_log = list(fctx.filelog())
             if limit:
                 # Limit to the last n items
                 file_log = file_log[-limit:]
 
             return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
         return _node_history_until(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(path)
 
         result = []
         for i, annotate_obj in enumerate(fctx.annotate(), 1):
             ln_no = i
             sha = hex(annotate_obj.fctx.node())
             content = annotate_obj.text
             result.append((ln_no, sha, content))
         return result
 
     @reraise_safe_exceptions
     def fctx_node_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(path)
         return fctx.data()
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_flags(_repo_id, _commit_id, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             fctx = ctx.filectx(path)
             return fctx.flags()
 
         return _fctx_flags(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def fctx_size(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _fctx_size(_repo_id, _revision, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, commit_id)
             fctx = ctx.filectx(path)
             return fctx.size()
         return _fctx_size(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_all_commit_ids(_context_uid, _repo_id, _name):
             repo = self._factory.repo(wire)
             repo = repo.filtered(name)
             revs = map(lambda x: hex(x[7]), repo.changelog.index)
             return revs
         return _get_all_commit_ids(context_uid, repo_id, name)
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, commit_id, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
             return largefiles.lfutil.isstandin(path)
 
         return _is_large_file(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def is_binary(self, wire, revision, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _sha, _path):
             repo = self._factory.repo(wire)
             ctx = self._get_ctx(repo, revision)
             fctx = ctx.filectx(path)
             return fctx.isbinary()
 
         return _is_binary(repo_id, revision, path)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _lookup(_context_uid, _repo_id, _revision, _both):
 
             repo = self._factory.repo(wire)
             rev = _revision
             if isinstance(rev, int):
                 # NOTE(marcink):
                 # since Mercurial doesn't support negative indexes properly
                 # we need to shift accordingly by one to get proper index, e.g
                 # repo[-1] => repo[-2]
                 # repo[0]  => repo[-1]
                 if rev <= 0:
                     rev = rev + -1
             try:
                 ctx = self._get_ctx(repo, rev)
             except (TypeError, RepoLookupError) as e:
                 e._org_exc_tb = traceback.format_exc()
                 raise exceptions.LookupException(e)(rev)
             except LookupError as e:
                 e._org_exc_tb = traceback.format_exc()
                 raise exceptions.LookupException(e)(e.name)
 
             if not both:
                 return ctx.hex()
 
             ctx = repo[ctx.hex()]
             return ctx.hex(), ctx.rev()
 
         return _lookup(context_uid, repo_id, revision, both)
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url):
         if not self.check_url(url, wire['config']):
             return
 
         repo = self._factory.repo(wire)
 
         # Disable any prompts for this repo
         repo.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         bookmarks = dict(repo._bookmarks).keys()
         remote = peer(repo, {}, url)
         # Disable any prompts for this remote
         remote.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         return exchange.push(
             repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, rev)
         return ctx.rev()
 
     @reraise_safe_exceptions
     def rev_range(self, wire, commit_filter):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _rev_range(_context_uid, _repo_id, _filter):
             repo = self._factory.repo(wire)
             revisions = [rev for rev in revrange(repo, commit_filter)]
             return revisions
 
         return _rev_range(context_uid, repo_id, sorted(commit_filter))
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         other_path = kwargs.pop('other_path', None)
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
 
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
         verify.verify(repo)
         return output.getvalue()
 
     @reraise_safe_exceptions
     def hg_update_cache(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
         with repo.wlock(), repo.lock():
             repo.updatecaches(full=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def hg_rebuild_fn_cache(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui, output = patch_ui_message_output(baseui)
 
         repo.ui = baseui
 
         repair.rebuildfncache(baseui, repo)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def tags(self, wire):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _tags(_context_uid, _repo_id):
             repo = self._factory.repo(wire)
             return repo.tags()
 
         return _tags(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def update(self, wire, node=None, clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [branch]
         else:
             args = []
         commands.heads(baseui, repo, template='{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
         a = changelog.ancestor(lookup(revision1), lookup(revision2))
         return hex(a)
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
         clone(baseui, source, dest, noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
 
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         publishing = baseui.configbool('phases', 'publish')
         if publishing:
             new_commit = 'public'
         else:
             new_commit = 'draft'
 
         def _filectxfn(_repo, ctx, path):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if path in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
                 if node['path'] == path:
                     return memfilectx(
                         _repo,
                         changectx=ctx,
                         path=node['path'],
                         data=node['content'],
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copysource=False)
 
             raise exceptions.AbortException()(
                 "Given path haven't been marked as added, "
                 "changed or removed (%s)" % path)
 
         with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
 
             commit_ctx = memctx(
                 repo=repo,
                 parents=parents,
                 text=message,
                 files=files,
                 filectxfn=_filectxfn,
                 user=user,
                 date=(commit_time, commit_timezone),
                 extra=extra)
 
             n = repo.commitctx(commit_ctx)
             new_id = hex(n)
 
             return new_id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         # Disable any prompts for this repo
         repo.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         remote = peer(repo, {}, url)
         # Disable any prompts for this remote
         remote.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
 
         return exchange.pull(
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
     def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
         if bookmark:
             opts['bookmark'] = bookmark
         if branch:
             opts['branch'] = branch
         if revision:
             opts['rev'] = revision
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
     def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
         commands.push(baseui, repo, dest=dest_path, rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         hgext_strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
     def get_unresolved_files(self, wire):
         repo = self._factory.repo(wire)
 
         log.debug('Calculating unresolved files for repo: %s', repo)
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui = self._factory._create_config(wire['config'])
         baseui.write = write
 
         commands.resolve(baseui, repo, list=True)
         unresolved = output.getvalue().splitlines(0)
         return unresolved
 
     @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions
     def merge_state(self, wire):
         repo = self._factory.repo(wire)
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
         ms = hg_merge.mergestate(repo)
         return [x for x in ms.unresolved()]
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username, close_branch=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'username', username)
         commands.commit(baseui, repo, message=message, close_branch=close_branch)
 
     @reraise_safe_exceptions
     def rebase(self, wire, source=None, dest=None, abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
         rebase.rebase(baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         node = ctx.node()
 
         date = (tag_time, tag_timezone)
         try:
             hg_tag.tag(repo, name, node, message, local, user, date)
         except Abort as e:
             log.exception("Tag operation aborted")
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         # we don't need any special hooks for Mercurial
         pass
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         return {
             'pre_version': vcsserver.__version__,
             'post_version': vcsserver.__version__,
         }

vcsserver/hgcompat.py

@@ -1,79 +1,79 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """
 Mercurial libs compatibility
 """
 
 import mercurial
 from mercurial import demandimport
 # patch demandimport, due to bug in mercurial when it always triggers
 # demandimport.enable()
 demandimport.enable = lambda *args, **kwargs: 1
 
 from mercurial import ui
 from mercurial import patch
 from mercurial import config
 from mercurial import extensions
 from mercurial import scmutil
 from mercurial import archival
 from mercurial import discovery
 from mercurial import unionrepo
 from mercurial import localrepo
 from mercurial import merge as hg_merge
 from mercurial import subrepo
 from mercurial import subrepoutil
 from mercurial import tags as hg_tag
 
 from mercurial.commands import clone, nullid, pull
 from mercurial.context import memctx, memfilectx
 from mercurial.error import (
     LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
     RequirementError, ProgrammingError)
 from mercurial.hgweb import hgweb_mod
 from mercurial.localrepo import instance
 from mercurial.match import match
 from mercurial.mdiff import diffopts
 from mercurial.node import bin, hex
 from mercurial.encoding import tolocal
 from mercurial.discovery import findcommonoutgoing
 from mercurial.hg import peer
 from mercurial.httppeer import makepeer
 from mercurial.util import url as hg_url
 from mercurial.scmutil import revrange, revsymbol
 from mercurial.node import nullrev
 from mercurial import exchange
 from hgext import largefiles
 
 # those authnadlers are patched for python 2.6.5 bug an
 # infinit looping when given invalid resources
 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
 
 
 def get_ctx(repo, ref):
     try:
         ctx = repo[ref]
     except ProgrammingError:
         # we're unable to find the rev using a regular lookup, we fallback
         # to slower, but backward compat revsymbol usage
         ctx = revsymbol(repo, ref)
     except (LookupError, RepoLookupError):
         # Similar case as above but only for refs that are not numeric
         if isinstance(ref, (int, long)):
             raise
         ctx = revsymbol(repo, ref)
     return ctx

vcsserver/hgpatches.py

@@ -1,134 +1,134 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """
 Adjustments to Mercurial
 
 Intentionally kept separate from `hgcompat` and `hg`, so that these patches can
 be applied without having to import the whole Mercurial machinery.
 
 Imports are function local, so that just importing this module does not cause
 side-effects other than these functions being defined.
 """
 
 import logging
 
 
 def patch_largefiles_capabilities():
     """
     Patches the capabilities function in the largefiles extension.
     """
     from vcsserver import hgcompat
     lfproto = hgcompat.largefiles.proto
     wrapper = _dynamic_capabilities_wrapper(
         lfproto, hgcompat.extensions.extensions)
     lfproto._capabilities = wrapper
 
 
 def _dynamic_capabilities_wrapper(lfproto, extensions):
 
     wrapped_capabilities = lfproto._capabilities
     logger = logging.getLogger('vcsserver.hg')
 
     def _dynamic_capabilities(orig, repo, proto):
         """
         Adds dynamic behavior, so that the capability is only added if the
         extension is enabled in the current ui object.
         """
         if 'largefiles' in dict(extensions(repo.ui)):
             logger.debug('Extension largefiles enabled')
             calc_capabilities = wrapped_capabilities
             return calc_capabilities(orig, repo, proto)
         else:
             logger.debug('Extension largefiles disabled')
             return orig(repo, proto)
 
     return _dynamic_capabilities
 
 
 def patch_subrepo_type_mapping():
     from collections import defaultdict
     from hgcompat import subrepo, subrepoutil
     from vcsserver.exceptions import SubrepoMergeException
 
     class NoOpSubrepo(subrepo.abstractsubrepo):
 
         def __init__(self, ctx, path, *args, **kwargs):
             """Initialize abstractsubrepo part
 
             ``ctx`` is the context referring this subrepository in the
             parent repository.
 
             ``path`` is the path to this subrepository as seen from
             innermost repository.
             """
             self.ui = ctx.repo().ui
             self._ctx = ctx
             self._path = path
 
         def storeclean(self, path):
             """
             returns true if the repository has not changed since it was last
             cloned from or pushed to a given repository.
             """
             return True
 
         def dirty(self, ignoreupdate=False, missing=False):
             """returns true if the dirstate of the subrepo is dirty or does not
             match current stored state. If ignoreupdate is true, only check
             whether the subrepo has uncommitted changes in its dirstate.
             """
             return False
 
         def basestate(self):
             """current working directory base state, disregarding .hgsubstate
             state and working directory modifications"""
             substate = subrepoutil.state(self._ctx, self.ui)
             file_system_path, rev, repotype = substate.get(self._path)
             return rev
 
         def remove(self):
             """remove the subrepo
 
             (should verify the dirstate is not dirty first)
             """
             pass
 
         def get(self, state, overwrite=False):
             """run whatever commands are needed to put the subrepo into
             this state
             """
             pass
 
         def merge(self, state):
             """merge currently-saved state with the new state."""
             raise SubrepoMergeException()()
 
         def push(self, opts):
             """perform whatever action is analogous to 'hg push'
 
             This may be a no-op on some systems.
             """
             pass
 
     # Patch subrepo type mapping to always return our NoOpSubrepo class
     # whenever a subrepo class is looked up.
     subrepo.types = {
         'hg': NoOpSubrepo,
         'git': NoOpSubrepo,
         'svn': NoOpSubrepo
     }

vcsserver/hook_utils/__init__.py

@@ -1,205 +1,205 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import re
 import os
 import sys
 import datetime
 import logging
 import pkg_resources
 
 import vcsserver
 
 log = logging.getLogger(__name__)
 
 
 def get_git_hooks_path(repo_path, bare):
     hooks_path = os.path.join(repo_path, 'hooks')
     if not bare:
         hooks_path = os.path.join(repo_path, '.git', 'hooks')
 
     return hooks_path
 
 
 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
     """
     Creates a RhodeCode hook inside a git repository
 
     :param repo_path: path to repository
     :param executable: binary executable to put in the hooks
     :param force_create: Create even if same name hook exists
     """
     executable = executable or sys.executable
     hooks_path = get_git_hooks_path(repo_path, bare)
 
     if not os.path.isdir(hooks_path):
         os.makedirs(hooks_path, mode=0o777)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
     tmpl_pre = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
 
     path = ''  # not used for now
     timestamp = datetime.datetime.utcnow().isoformat()
 
     for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
         log.debug('Installing git hook in repo %s', repo_path)
         _hook_file = os.path.join(hooks_path, '%s-receive' % h_type)
         _rhodecode_hook = check_rhodecode_hook(_hook_file)
 
         if _rhodecode_hook or force_create:
             log.debug('writing git %s hook file at %s !', h_type, _hook_file)
             try:
                 with open(_hook_file, 'wb') as f:
                     template = template.replace(
                         '_TMPL_', vcsserver.__version__)
                     template = template.replace('_DATE_', timestamp)
                     template = template.replace('_ENV_', executable)
                     template = template.replace('_PATH_', path)
                     f.write(template)
                 os.chmod(_hook_file, 0o755)
             except IOError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
 
     return True
 
 
 def get_svn_hooks_path(repo_path):
     hooks_path = os.path.join(repo_path, 'hooks')
 
     return hooks_path
 
 
 def install_svn_hooks(repo_path, executable=None, force_create=False):
     """
     Creates RhodeCode hooks inside a svn repository
 
     :param repo_path: path to repository
     :param executable: binary executable to put in the hooks
     :param force_create: Create even if same name hook exists
     """
     executable = executable or sys.executable
     hooks_path = get_svn_hooks_path(repo_path)
     if not os.path.isdir(hooks_path):
         os.makedirs(hooks_path, mode=0o777)
 
     tmpl_post = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
     tmpl_pre = pkg_resources.resource_string(
         'vcsserver', '/'.join(
             ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
 
     path = ''  # not used for now
     timestamp = datetime.datetime.utcnow().isoformat()
 
     for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
         log.debug('Installing svn hook in repo %s', repo_path)
         _hook_file = os.path.join(hooks_path, '%s-commit' % h_type)
         _rhodecode_hook = check_rhodecode_hook(_hook_file)
 
         if _rhodecode_hook or force_create:
             log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
 
             try:
                 with open(_hook_file, 'wb') as f:
                     template = template.replace(
                         '_TMPL_', vcsserver.__version__)
                     template = template.replace('_DATE_', timestamp)
                     template = template.replace('_ENV_', executable)
                     template = template.replace('_PATH_', path)
 
                     f.write(template)
                 os.chmod(_hook_file, 0o755)
             except IOError:
                 log.exception('error writing hook file %s', _hook_file)
         else:
             log.debug('skipping writing hook file')
 
     return True
 
 
 def get_version_from_hook(hook_path):
     version = ''
     hook_content = read_hook_content(hook_path)
     matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
     if matches:
         try:
             version = matches.groups()[0]
             log.debug('got version %s from hooks.', version)
         except Exception:
             log.exception("Exception while reading the hook version.")
     return version.replace("'", "")
 
 
 def check_rhodecode_hook(hook_path):
     """
     Check if the hook was created by RhodeCode
     """
     if not os.path.exists(hook_path):
         return True
 
     log.debug('hook exists, checking if it is from RhodeCode')
 
     version = get_version_from_hook(hook_path)
     if version:
         return True
 
     return False
 
 
 def read_hook_content(hook_path):
     content = ''
     if os.path.isfile(hook_path):
         with open(hook_path, 'rb') as f:
             content = f.read()
     return content
 
 
 def get_git_pre_hook_version(repo_path, bare):
     hooks_path = get_git_hooks_path(repo_path, bare)
     _hook_file = os.path.join(hooks_path, 'pre-receive')
     version = get_version_from_hook(_hook_file)
     return version
 
 
 def get_git_post_hook_version(repo_path, bare):
     hooks_path = get_git_hooks_path(repo_path, bare)
     _hook_file = os.path.join(hooks_path, 'post-receive')
     version = get_version_from_hook(_hook_file)
     return version
 
 
 def get_svn_pre_hook_version(repo_path):
     hooks_path = get_svn_hooks_path(repo_path)
     _hook_file = os.path.join(hooks_path, 'pre-commit')
     version = get_version_from_hook(_hook_file)
     return version
 
 
 def get_svn_post_hook_version(repo_path):
     hooks_path = get_svn_hooks_path(repo_path)
     _hook_file = os.path.join(hooks_path, 'post-commit')
     version = get_version_from_hook(_hook_file)
     return version

vcsserver/hooks.py

@@ -1,729 +1,729 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import logging
 import collections
 import importlib
 import base64
 
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import simplejson as json
 
 from vcsserver import exceptions, subprocessio, settings
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         try:
             connection.request('POST', '/', body)
         except Exception:
             log.error('Connection failed on %s', connection)
             raise
         response = connection.getresponse()
 
         response_data = response.read()
 
         try:
             return json.loads(response_data)
         except Exception:
             log.exception('Failed to decode hook response json data. '
                           'response_code:%s, raw_data:%s',
                           response.status, response_data)
             raise
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class HooksShadowRepoClient(object):
 
     def __call__(self, hook_name, extras):
         return {'output': '', 'status': 0}
 
 
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 class SvnMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to svn clients."""
 
     def __init__(self, stderr=None):
         # SVN needs data sent to stderr for back-to-client messaging
         self.stderr = stderr or sys.stderr
 
     def write(self, message):
         self.stderr.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
     hooks_uri = extras.get('hooks_uri')
     is_shadow_repo = extras.get('is_shadow_repo')
     if hooks_uri:
         return HooksHttpClient(extras['hooks_uri'])
     elif is_shadow_repo:
         return HooksShadowRepoClient()
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks_client = _get_hooks_client(extras)
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
 
     _handle_exception(result)
     writer.write(result['output'])
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node, check_heads=False):
     from vcsserver.hgcompat import get_ctx
 
     commits = []
     revs = []
     start = get_ctx(repo, node).rev()
     end = len(repo)
     for rev in range(start, end):
         revs.append(rev)
         ctx = get_ctx(repo, rev)
         commit_id = mercurial.node.hex(ctx.node())
         branch = ctx.branch()
         commits.append((commit_id, branch))
 
     parent_heads = []
     if check_heads:
         parent_heads = _check_heads(repo, start, end, revs)
     return commits, parent_heads
 
 
 def _check_heads(repo, start, end, commits):
     from vcsserver.hgcompat import get_ctx
     changelog = repo.changelog
     parents = set()
 
     for new_rev in commits:
         for p in changelog.parentrevs(new_rev):
             if p == mercurial.node.nullrev:
                 continue
             if p < start:
                 parents.add(p)
 
     for p in parents:
         branch = get_ctx(repo, p).branch()
         # The heads descending from that parent, on the same branch
         parent_heads = set([p])
         reachable = set([p])
         for x in xrange(p + 1, end):
             if get_ctx(repo, x).branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
                 if pp in reachable:
                     reachable.add(x)
                     parent_heads.discard(pp)
                     parent_heads.add(x)
         # More than one head? Suggest merging
         if len(parent_heads) > 1:
             return list(parent_heads)
 
     return []
 
 
 def _get_git_env():
     env = {}
     for k, v in os.environ.items():
         if k.startswith('GIT'):
             env[k] = v
 
     # serialized version
     return [(k, v) for k, v in env.items()]
 
 
 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
     env = {}
     for k, v in os.environ.items():
         if k.startswith('HG'):
             env[k] = v
 
     env['HG_NODE'] = old_rev
     env['HG_NODE_LAST'] = new_rev
     env['HG_TXNID'] = txnid
     env['HG_PENDING'] = repo_path
 
     return [(k, v) for k, v in env.items()]
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook
     """
     extras = _extras_from_ui(ui)
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
     if node and kwargs.get('hooktype') == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
             old_rev = kwargs.get('node_last') or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
         for push_ref in rev_data:
             push_ref['multiple_heads'] = _heads
 
             repo_path = os.path.join(
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
                 new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
                 repo_path=repo_path)
 
     extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook for SSH
     """
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook
     """
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_branches'):
         bookmarks = ui._rc_pushkey_branches
 
     extras['hook_type'] = kwargs.get('hooktype', 'post_push')
     extras['commit_ids'] = commit_ids
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook for SSH
     """
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
     from vcsserver.hgcompat import get_ctx
     if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_branches = get_ctx(repo, kwargs['key']).bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 # NOTE(marcink):
                 # we're unable to tell total_commits for git at this point
                 # but we set the variable for consistency with GIT
                 'total_commits': -1,
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     empty_commit_id = '0' * 40
 
     detect_force_push = extras.get('detect_force_push')
 
     for push_ref in rev_data:
         # store our git-env which holds the temp store
         push_ref['git_env'] = _get_git_env()
         push_ref['pruned_sha'] = ''
         if not detect_force_push:
             # don't check for forced-push when we don't need to
             continue
 
         type_ = push_ref['type']
         new_branch = push_ref['old_rev'] == empty_commit_id
         delete_branch = push_ref['new_rev'] == empty_commit_id
         if type_ == 'heads' and not (new_branch or delete_branch):
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
             cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, '^{}'.format(new_rev)]
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push was used
             if stdout:
                 push_ref['pruned_sha'] = stdout.splitlines()
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
                 # starting new branch case
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 # Fix up head revision if needed
                 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
                 try:
                     subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
                     cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
                     print("Setting default branch to %s" % push_ref['name'])
                     subprocessio.run_command(cmd, env=os.environ.copy())
 
                 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
                        '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 heads = stdout
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
                 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
                        '--pretty=format:%H', '--', push_ref['new_rev'],
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE, 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['hook_type'] = 'post_receive'
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())
 
 
 def _get_extras_from_txn_id(path, txn_id):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-t', txn_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from txn_id')
 
     return extras
 
 
 def _get_extras_from_commit_id(commit_id, path):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-r', commit_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from commit_id')
 
     return extras
 
 
 def svn_pre_commit(repo_path, commit_data, env):
     path, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_txn_id(path, txn_id)
         if not extras:
             return 0
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = [txn_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'total_commits': 1,
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     return _call_hook('pre_push', extras, SvnMessageWriter())
 
 
 def svn_post_commit(repo_path, commit_data, env):
     """
     commit_data is path, rev, txn_id
     """
     if len(commit_data) == 3:
         path, commit_id, txn_id = commit_data
     elif len(commit_data) == 2:
         log.error('Failed to extract txn_id from commit_data using legacy method. '
                   'Some functionality might be limited')
         path, commit_id = commit_data
         txn_id = None
 
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_commit_id(commit_id, path)
         if not extras:
             return 0
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
         'total_commits': 1,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, SvnMessageWriter())
         except Exception:
             pass
 
     return _call_hook('post_push', extras, SvnMessageWriter())

vcsserver/http_main.py

@@ -1,675 +1,688 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import sys
 import base64
 import locale
 import logging
 import uuid
 import wsgiref.util
 import traceback
 import tempfile
 from itertools import chain
 from cStringIO import StringIO
 
 import simplejson as json
 import msgpack
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.compat import configparser
 from pyramid.response import Response
 
 from vcsserver.utils import safe_int
 
 log = logging.getLogger(__name__)
 
 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
 
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
     log.error(
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
 
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
 
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 
 def _is_request_chunked(environ):
     stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
     return stream
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
     return settings[name]
 
 
 def _bool_setting(settings, name, default):
     input_val = settings.get(name, default)
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
     return settings[name]
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     # Otherwise we assume it uses pyramids space/newline separation.
     settings[name] = aslist(raw_value)
     return settings[name]
 
 
 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
 
     if default_when_empty and not value:
         # use default value when value is empty
         value = default
 
     if lower:
         value = value.lower()
     settings[name] = value
     return settings[name]
 
 
 class VCS(object):
     def __init__(self, locale_conf=None, cache_config=None):
         self.locale = locale_conf
         self.cache_config = cache_config
         self._configure_locale()
 
         if GitFactory and GitRemote:
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
 
             # hg factory is used for svn url validation
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.info("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 def not_found(request):
     return {'status': '404 NOT FOUND'}
 
 
 class VCSViewPredicate(object):
     def __init__(self, val, config):
         self.remotes = val
 
     def text(self):
         return 'vcs view method = %s' % (self.remotes.keys(),)
 
     phash = text
 
     def __call__(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self.remotes
 
 
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
         self._sanitize_settings_and_apply_defaults(settings)
 
         self.config = Configurator(settings=settings)
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
 
         settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale_conf=settings_locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
         self._configure_settings(global_config, settings)
         self._configure()
 
     def _configure_settings(self, global_config, app_settings):
         """
         Configure the settings module.
         """
         settings_merged = global_config.copy()
         settings_merged.update(app_settings)
 
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
         binary_dir = app_settings.get('core.binary_dir', None)
         if binary_dir:
             settings.BINARY_DIR = binary_dir
 
         # Store the settings to make them available to other modules.
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
     def _sanitize_settings_and_apply_defaults(self, settings):
         temp_store = tempfile.gettempdir()
         default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
         # save default, cache dir, and use it for all backends later.
         default_cache_dir = _string_setting(
             settings,
             'cache_dir',
             default_cache_dir, lower=False, default_when_empty=True)
 
         # ensure we have our dir created
         if not os.path.isdir(default_cache_dir):
             os.makedirs(default_cache_dir, mode=0o755)
 
         # exception store cache
         _string_setting(
             settings,
             'exception_tracker.store_path',
             temp_store, lower=False, default_when_empty=True)
 
         # repo_object cache
         _string_setting(
             settings,
             'rc_cache.repo_object.backend',
             'dogpile.cache.rc.file_namespace', lower=False)
         _int_setting(
             settings,
             'rc_cache.repo_object.expiration_time',
             30 * 24 * 60 * 60)
         _string_setting(
             settings,
             'rc_cache.repo_object.arguments.filename',
             os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
 
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
 
         # rpc methods
         self.config.add_route('vcs', '/{backend}')
 
         # streaming rpc remote methods
         self.config.add_route('vcs_stream', '/{backend}/stream')
 
         # vcs operations clone/push as streaming
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(self.status_view, route_name='status', renderer='json')
         self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
                              vcs_view=self._remotes)
         self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
                              vcs_view=self._remotes)
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         self.config.add_view_predicate('vcs_view', VCSViewPredicate)
 
         self.config.add_notfound_view(not_found, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.request_wrapper.RequestWrapperTween',
         )
         self.config.add_request_method(
             'vcsserver.lib.request_counter.get_request_counter',
             'request_count')
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def _vcs_view_params(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
         params = payload['params']
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         context_uid = None
 
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
         repo_state_uid = wire.get('repo_state_uid') if wire else None
 
         # NOTE(marcink): trading complexity for slight performance
         if log.isEnabledFor(logging.DEBUG):
             no_args_methods = [
                 'archive_repo'
             ]
             if method in no_args_methods:
                 call_args = ''
             else:
                 call_args = args[1:]
 
             log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
                       method, call_args, kwargs, context_uid, repo_state_uid)
 
         return payload, remote, method, args, kwargs
 
     def vcs_view(self, request):
 
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         payload_id = payload.get('id')
 
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             exc_info = list(sys.exc_info())
             exc_type, exc_value, exc_traceback = exc_info
 
             org_exc = getattr(e, '_org_exc', None)
             org_exc_name = None
             org_exc_tb = ''
             if org_exc:
                 org_exc_name = org_exc.__class__.__name__
                 org_exc_tb = getattr(e, '_org_exc_tb', '')
                 # replace our "faked" exception with our org
                 exc_info[0] = org_exc.__class__
                 exc_info[1] = org_exc
 
+            should_store_exc = True
+            if org_exc:
+                def get_exc_fqn(_exc_obj):
+                    module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
+                    return module_name + '.' + org_exc_name
+
+                exc_fqn = get_exc_fqn(org_exc)
+
+                if exc_fqn in ['mercurial.error.RepoLookupError',
+                               'vcsserver.exceptions.RefNotFoundException']:
+                    should_store_exc = False
+
+            if should_store_exc:
             store_exception(id(exc_info), exc_info)
 
             tb_info = ''.join(
                 traceback.format_exception(exc_type, exc_value, exc_traceback))
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload_id,
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'org_exc_tb': org_exc_tb,
                     'type': type_
                 }
             }
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload_id,
                 'result': resp
             }
 
         return resp
 
     def vcs_stream_view(self, request):
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         # this method has a stream: marker we remove it here
         method = method.split('stream:')[-1]
         chunk_size = safe_int(payload.get('chunk_size')) or 4096
 
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             raise
 
         def get_chunked_data(method_resp):
             stream = StringIO(method_resp)
             while 1:
                 chunk = stream.read(chunk_size)
                 if not chunk:
                     break
                 yield chunk
 
         response = Response(app_iter=get_chunked_data(resp))
         response.content_type = 'application/octet-stream'
 
         return response
 
     def status_view(self, request):
         import vcsserver
         return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
                 'pid': os.getpid()}
 
     def service_view(self, request):
         import vcsserver
 
         payload = msgpack.unpackb(request.body, use_list=True)
         server_config, app_config = {}, {}
 
         try:
             path = self.global_config['__file__']
             config = configparser.RawConfigParser()
 
             config.read(path)
 
             if config.has_section('server:main'):
                 server_config = dict(config.items('server:main'))
             if config.has_section('app:main'):
                 app_config = dict(config.items('app:main'))
 
         except Exception:
             log.exception('Failed to read .ini file for display')
 
         environ = os.environ.items()
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
                 config=server_config,
                 app_config=app_config,
                 environ=environ,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
             return msgpack.packb(value)
         return _render
 
     def set_env_from_config(self, environ, config):
         dict_conf = {}
         try:
             for elem in config:
                 if elem[0] == 'rhodecode':
                     dict_conf = json.loads(elem[2])
                     break
         except Exception:
             log.exception('Failed to fetch SCM CONFIG')
             return
 
         username = dict_conf.get('username')
         if username:
             environ['REMOTE_USER'] = username
             # mercurial specific, some extension api rely on this
             environ['HGUSER'] = username
 
         ip = dict_conf.get('ip')
         if ip:
             environ['REMOTE_HOST'] = ip
 
         if _is_request_chunked(environ):
             # set the compatibility flag for webob
             environ['wsgi.input_terminated'] = True
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
 
                 path = environ['PATH_INFO']
                 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
                 log.debug(
                     'LFS: Detecting if request `%s` is LFS server path based '
                     'on content type:`%s`, is_lfs:%s',
                     path, content_type, is_lfs_request)
 
                 if not is_lfs_request:
                     # fallback detection by path
                     if GIT_LFS_PROTO_PAT.match(path):
                         is_lfs_request = True
                     log.debug(
                         'LFS: fallback detection by path of: `%s`, is_lfs:%s',
                         path, is_lfs_request)
 
                 if is_lfs_request:
                     app = scm_app.create_git_lfs_wsgi_app(
                         repo_path, repo_name, config)
                 else:
                     app = scm_app.create_git_wsgi_app(
                         repo_path, repo_name, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
 
                 return app(environ, start_response)
 
             return _git_stream
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
         if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
             return HTTPRepoBranchProtected(title=exception.message)
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
 
         traceback_info = 'unavailable'
         if request.exc_info:
             exc_type, exc_value, exc_tb = request.exc_info
             traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
 
         log.error(
             'error occurred handling this request for path: %s, \n tb: %s',
             request.path, traceback_info)
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
     app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/lib/__init__.py

@@ -1,16 +1,16 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/lib/exc_tracking.py

@@ -1,169 +1,169 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 import os
 import time
 import datetime
 import msgpack
 import logging
 import traceback
 import tempfile
 
 from pyramid import compat
 
 log = logging.getLogger(__name__)
 
 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
 global_prefix = 'vcsserver'
 exc_store_dir_name = 'rc_exception_store_v1'
 
 
 def exc_serialize(exc_id, tb, exc_type):
 
     data = {
         'version': 'v1',
         'exc_id': exc_id,
         'exc_utc_date': datetime.datetime.utcnow().isoformat(),
         'exc_timestamp': repr(time.time()),
         'exc_message': tb,
         'exc_type': exc_type,
     }
     return msgpack.packb(data), data
 
 
 def exc_unserialize(tb):
     return msgpack.unpackb(tb)
 
 
 def get_exc_store():
     """
     Get and create exception store if it's not existing
     """
     import vcsserver as app
 
     exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
     _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
 
     _exc_store_path = os.path.abspath(_exc_store_path)
     if not os.path.isdir(_exc_store_path):
         os.makedirs(_exc_store_path)
         log.debug('Initializing exceptions store at %s', _exc_store_path)
     return _exc_store_path
 
 
 def _store_exception(exc_id, exc_info, prefix):
     exc_type, exc_value, exc_traceback = exc_info
 
     tb = ''.join(traceback.format_exception(
         exc_type, exc_value, exc_traceback, None))
 
     detailed_tb = getattr(exc_value, '_org_exc_tb', None)
 
     if detailed_tb:
         if isinstance(detailed_tb, compat.string_types):
             remote_tb = [detailed_tb]
 
         tb += (
             '\n+++ BEG SOURCE EXCEPTION +++\n\n'
             '{}\n'
             '+++ END SOURCE EXCEPTION +++\n'
             ''.format('\n'.join(remote_tb))
         )
 
         # Avoid that remote_tb also appears in the frame
         del remote_tb
 
     exc_type_name = exc_type.__name__
     exc_store_path = get_exc_store()
     exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
     exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
     if not os.path.isdir(exc_store_path):
         os.makedirs(exc_store_path)
     stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
     with open(stored_exc_path, 'wb') as f:
         f.write(exc_data)
     log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
 
 
 def store_exception(exc_id, exc_info, prefix=global_prefix):
     """
     Example usage::
 
         exc_info = sys.exc_info()
         store_exception(id(exc_info), exc_info)
     """
 
     try:
         _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
     except Exception:
         log.exception('Failed to store exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass
 
 
 def _find_exc_file(exc_id, prefix=global_prefix):
     exc_store_path = get_exc_store()
     if prefix:
         exc_id = '{}_{}'.format(exc_id, prefix)
     else:
         # search without a prefix
         exc_id = '{}'.format(exc_id)
 
     # we need to search the store for such start pattern as above
     for fname in os.listdir(exc_store_path):
         if fname.startswith(exc_id):
             exc_id = os.path.join(exc_store_path, fname)
             break
         continue
     else:
         exc_id = None
 
     return exc_id
 
 
 def _read_exception(exc_id, prefix):
     exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
     if exc_id_file_path:
         with open(exc_id_file_path, 'rb') as f:
             return exc_unserialize(f.read())
     else:
         log.debug('Exception File `%s` not found', exc_id_file_path)
     return None
 
 
 def read_exception(exc_id, prefix=global_prefix):
     try:
         return _read_exception(exc_id=exc_id, prefix=prefix)
     except Exception:
         log.exception('Failed to read exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
     return None
 
 
 def delete_exception(exc_id, prefix=global_prefix):
     try:
         exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
         if exc_id_file_path:
             os.remove(exc_id_file_path)
 
     except Exception:
         log.exception('Failed to remove exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass

vcsserver/lib/memory_lru_dict.py

@@ -1,65 +1,65 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 import logging
 
 from repoze.lru import LRUCache
 
 from vcsserver.utils import safe_str
 
 log = logging.getLogger(__name__)
 
 
 class LRUDict(LRUCache):
     """
     Wrapper to provide partial dict access
     """
 
     def __setitem__(self, key, value):
         return self.put(key, value)
 
     def __getitem__(self, key):
         return self.get(key)
 
     def __contains__(self, key):
         return bool(self.get(key))
 
     def __delitem__(self, key):
         del self.data[key]
 
     def keys(self):
         return self.data.keys()
 
 
 class LRUDictDebug(LRUDict):
     """
     Wrapper to provide some debug options
     """
     def _report_keys(self):
         elems_cnt = '%s/%s' % (len(self.keys()), self.size)
         # trick for pformat print it more nicely
         fmt = '\n'
         for cnt, elem in enumerate(self.keys()):
             fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
         log.debug('current LRU keys (%s):%s', elems_cnt, fmt)
 
     def __getitem__(self, key):
         self._report_keys()
         return self.get(key)

vcsserver/lib/rc_cache/__init__.py

@@ -1,72 +1,72 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 from dogpile.cache import register_backend
 
 register_backend(
     "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
 register_backend(
     "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
     "FileNamespaceBackend")
 
 register_backend(
     "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
     "RedisPickleBackend")
 
 register_backend(
     "dogpile.cache.rc.redis_msgpack", "vcsserver.lib.rc_cache.backends",
     "RedisMsgPackBackend")
 
 
 log = logging.getLogger(__name__)
 
 from . import region_meta
 from .utils import (get_default_cache_settings, backend_key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         avail_regions.add(namespace_name)
     log.debug('dogpile: found following cache regions: %s', avail_regions)
 
     # register them into namespace
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
             function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
         new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
         log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

vcsserver/lib/rc_cache/backends.py

@@ -1,253 +1,253 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import time
 import errno
 import logging
 
 import msgpack
 import redis
 
 from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import redis as redis_backend
 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
 from dogpile.cache.util import memoized_property
 
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
     key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
 
 
 class PickleSerializer(object):
 
     def _dumps(self, value, safe=False):
         try:
             return compat.pickle.dumps(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
     def _loads(self, value, safe=True):
         try:
             return compat.pickle.loads(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
 
 class MsgPackSerializer(object):
 
     def _dumps(self, value, safe=False):
         try:
             return msgpack.packb(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
     def _loads(self, value, safe=True):
         """
         pickle maintained the `CachedValue` wrapper of the tuple
         msgpack does not, so it must be added back in.
        """
         try:
             value = msgpack.unpackb(value, use_list=False)
             return CachedValue(*value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
 
 import fcntl
 flock_org = fcntl.flock
 
 
 class CustomLockFactory(FileLock):
 
     pass
 
 
 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
     key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         super(FileNamespaceBackend, self).__init__(arguments)
 
     def __repr__(self):
         return '{} `{}`'.format(self.__class__, self.filename)
 
     def list_keys(self, prefix=''):
         prefix = '{}:{}'.format(self.key_prefix, prefix)
 
         def cond(v):
             if not prefix:
                 return True
 
             if v.startswith(prefix):
                 return True
             return False
 
         with self._dbm_file(True) as dbm:
 
             return filter(cond, dbm.keys())
 
     def get_store(self):
         return self.filename
 
     def get(self, key):
         with self._dbm_file(False) as dbm:
             if hasattr(dbm, 'get'):
                 value = dbm.get(key, NO_VALUE)
             else:
                 # gdbm objects lack a .get method
                 try:
                     value = dbm[key]
                 except KeyError:
                     value = NO_VALUE
             if value is not NO_VALUE:
                 value = self._loads(value)
             return value
 
     def set(self, key, value):
         with self._dbm_file(True) as dbm:
             dbm[key] = self._dumps(value)
 
     def set_multi(self, mapping):
         with self._dbm_file(True) as dbm:
             for key, value in mapping.items():
                 dbm[key] = self._dumps(value)
 
 
 class BaseRedisBackend(redis_backend.RedisBackend):
 
     def _create_client(self):
         args = {}
 
         if self.url is not None:
             args.update(url=self.url)
 
         else:
             args.update(
                 host=self.host, password=self.password,
                 port=self.port, db=self.db
             )
 
         connection_pool = redis.ConnectionPool(**args)
 
         return redis.StrictRedis(connection_pool=connection_pool)
 
     def list_keys(self, prefix=''):
         prefix = '{}:{}*'.format(self.key_prefix, prefix)
         return self.client.keys(prefix)
 
     def get_store(self):
         return self.client.connection_pool
 
     def get(self, key):
         value = self.client.get(key)
         if value is None:
             return NO_VALUE
         return self._loads(value)
 
     def get_multi(self, keys):
         if not keys:
             return []
         values = self.client.mget(keys)
         loads = self._loads
         return [
             loads(v) if v is not None else NO_VALUE
             for v in values]
 
     def set(self, key, value):
         if self.redis_expiration_time:
             self.client.setex(key, self.redis_expiration_time,
                               self._dumps(value))
         else:
             self.client.set(key, self._dumps(value))
 
     def set_multi(self, mapping):
         dumps = self._dumps
         mapping = dict(
             (k, dumps(v))
             for k, v in mapping.items()
         )
 
         if not self.redis_expiration_time:
             self.client.mset(mapping)
         else:
             pipe = self.client.pipeline()
             for key, value in mapping.items():
                 pipe.setex(key, self.redis_expiration_time, value)
             pipe.execute()
 
     def get_mutex(self, key):
         u = redis_backend.u
         if self.distributed_lock:
             lock_key = u('_lock_{0}').format(key)
             log.debug('Trying to acquire Redis lock for key %s', lock_key)
             return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
         else:
             return None
 
 
 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
     key_prefix = 'redis_pickle_backend'
     pass
 
 
 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
     key_prefix = 'redis_msgpack_backend'
     pass

vcsserver/lib/rc_cache/region_meta.py

@@ -1,26 +1,26 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import tempfile
 
 dogpile_config_defaults = {
     'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
 }
 
 # GLOBAL TO STORE ALL REGISTERED REGIONS
 dogpile_cache_regions = {}

vcsserver/lib/rc_cache/utils.py

@@ -1,153 +1,153 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import logging
 import functools
 from decorator import decorate
 
 from dogpile.cache import CacheRegion
 from dogpile.cache.util import compat
 
 from vcsserver.utils import safe_str, sha1
 
 
 log = logging.getLogger(__name__)
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=compat.string_type,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different than should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = compat.callable(expiration_time)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
         def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
 
             if not condition:
                 log.debug('Calling un-cached func:%s', user_func.func_name)
                 return user_func(*arg, **kw)
 
             key = key_generator(*arg, **kw)
 
             timeout = expiration_time() if expiration_time_is_callable \
                 else expiration_time
 
             log.debug('Calling cached fn:%s', user_func.func_name)
             return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
 
         def cache_decorator(user_func):
             if to_str is compat.string_type:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, user_func)
             else:
                 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
 
             def refresh(*arg, **kw):
                 """
                 Like invalidate, but regenerates the value instead
                 """
                 key = key_generator(*arg, **kw)
                 value = user_func(*arg, **kw)
                 self.set(key, value)
                 return value
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
             user_func.set = set_
             user_func.invalidate = invalidate
             user_func.get = get
             user_func.refresh = refresh
             user_func.key_generator = key_generator
             user_func.original = user_func
 
             # Use `decorate` to preserve the signature of :param:`user_func`.
 
             return decorate(user_func, functools.partial(
                 get_or_create_for_user_func, key_generator))
 
         return cache_decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
                 if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
 def backend_key_generator(backend):
     """
     Special wrapper that also sends over the backend to the key generator
     """
     def wrapper(namespace, fn):
         return key_generator(backend, namespace, fn)
     return wrapper
 
 
 def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
         backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
         namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
         final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key

vcsserver/lib/request_counter.py

@@ -1,27 +1,27 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 counter = 0
 
 
 def get_request_counter(request):
     global counter
     counter += 1
     return counter

vcsserver/pygrack.py

@@ -1,386 +1,386 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """Handles the Git smart protocol."""
 
 import os
 import socket
 import logging
 
 import simplejson as json
 import dulwich.protocol
 from webob import Request, Response, exc
 
 from vcsserver import hooks, subprocessio
 
 
 log = logging.getLogger(__name__)
 
 
 class FileWrapper(object):
     """File wrapper that ensures how much data is read from it."""
 
     def __init__(self, fd, content_length):
         self.fd = fd
         self.content_length = content_length
         self.remain = content_length
 
     def read(self, size):
         if size <= self.remain:
             try:
                 data = self.fd.read(size)
             except socket.error:
                 raise IOError(self)
             self.remain -= size
         elif self.remain:
             data = self.fd.read(self.remain)
             self.remain = 0
         else:
             data = None
         return data
 
     def __repr__(self):
         return '<FileWrapper %s len: %s, read: %s>' % (
             self.fd, self.content_length, self.content_length - self.remain
         )
 
 
 class GitRepository(object):
     """WSGI app for handling Git smart protocol endpoints."""
 
     git_folder_signature = frozenset(
         ('config', 'head', 'info', 'objects', 'refs'))
     commands = frozenset(('git-upload-pack', 'git-receive-pack'))
     valid_accepts = frozenset(('application/x-%s-result' %
                                c for c in commands))
 
     # The last bytes are the SHA1 of the first 12 bytes.
     EMPTY_PACK = (
         'PACK\x00\x00\x00\x02\x00\x00\x00\x00' +
         '\x02\x9d\x08\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
     )
     SIDE_BAND_CAPS = frozenset(('side-band', 'side-band-64k'))
 
     def __init__(self, repo_name, content_path, git_path, update_server_info,
                  extras):
         files = frozenset(f.lower() for f in os.listdir(content_path))
         valid_dir_signature = self.git_folder_signature.issubset(files)
 
         if not valid_dir_signature:
             raise OSError('%s missing git signature' % content_path)
 
         self.content_path = content_path
         self.repo_name = repo_name
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def _get_fixedpath(self, path):
         """
         Small fix for repo_path
 
         :param path:
         """
         path = path.split(self.repo_name, 1)[-1]
         if path.startswith('.git'):
             # for bare repos we still get the .git prefix inside, we skip it
             # here, and remove from the service command
             path = path[4:]
 
         return path.strip('/')
 
     def inforefs(self, request, unused_environ):
         """
         WSGI Response producer for HTTP GET Git Smart
         HTTP /info/refs request.
         """
 
         git_command = request.GET.get('service')
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPForbidden()
 
         # please, resist the urge to add '\n' to git capture and increment
         # line count by 1.
         # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
         # a part of protocol.
         # The code in Git client not only does NOT need '\n', but actually
         # blows up if you sprinkle "flush" (0000) as "0001\n".
         # It reads binary, per number of bytes specified.
         # if you do add '\n' as part of data, count it.
         server_advert = '# service=%s\n' % git_command
         packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
         try:
             gitenv = dict(os.environ)
             # forget all configs
             gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
             command = [self.git_path, git_command[4:], '--stateless-rpc',
                        '--advertise-refs', self.content_path]
             out = subprocessio.SubprocessIOChunker(
                 command,
                 env=gitenv,
                 starting_values=[packet_len + server_advert + '0000'],
                 shell=False
             )
         except EnvironmentError:
             log.exception('Error processing command')
             raise exc.HTTPExpectationFailed()
 
         resp = Response()
         resp.content_type = 'application/x-%s-advertisement' % str(git_command)
         resp.charset = None
         resp.app_iter = out
 
         return resp
 
     def _get_want_capabilities(self, request):
         """Read the capabilities found in the first want line of the request."""
         pos = request.body_file_seekable.tell()
         first_line = request.body_file_seekable.readline()
         request.body_file_seekable.seek(pos)
 
         return frozenset(
             dulwich.protocol.extract_want_line_capabilities(first_line)[1])
 
     def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
         """
         Construct a response with an empty PACK file.
 
         We use an empty PACK file, as that would trigger the failure of the pull
         or clone command.
 
         We also print in the error output a message explaining why the command
         was aborted.
 
         If aditionally, the user is accepting messages we send them the output
         of the pre-pull hook.
 
         Note that for clients not supporting side-band we just send them the
         emtpy PACK file.
         """
         if self.SIDE_BAND_CAPS.intersection(capabilities):
             response = []
             proto = dulwich.protocol.Protocol(None, response.append)
             proto.write_pkt_line('NAK\n')
             self._write_sideband_to_proto(pre_pull_messages, proto,
                                           capabilities)
             # N.B.(skreft): Do not change the sideband channel to 3, as that
             # produces a fatal error in the client:
             #   fatal: error in sideband demultiplexer
             proto.write_sideband(2, 'Pre pull hook failed: aborting\n')
             proto.write_sideband(1, self.EMPTY_PACK)
 
             # writes 0000
             proto.write_pkt_line(None)
 
             return response
         else:
             return [self.EMPTY_PACK]
 
     def _write_sideband_to_proto(self, data, proto, capabilities):
         """
         Write the data to the proto's sideband number 2.
 
         We do not use dulwich's write_sideband directly as it only supports
         side-band-64k.
         """
         if not data:
             return
 
         # N.B.(skreft): The values below are explained in the pack protocol
         # documentation, section Packfile Data.
         # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
         if 'side-band-64k' in capabilities:
             chunk_size = 65515
         elif 'side-band' in capabilities:
             chunk_size = 995
         else:
             return
 
         chunker = (
             data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size))
 
         for chunk in chunker:
             proto.write_sideband(2, chunk)
 
     def _get_messages(self, data, capabilities):
         """Return a list with packets for sending data in sideband number 2."""
         response = []
         proto = dulwich.protocol.Protocol(None, response.append)
 
         self._write_sideband_to_proto(data, proto, capabilities)
 
         return response
 
     def _inject_messages_to_response(self, response, capabilities,
                                      start_messages, end_messages):
         """
         Given a list response we inject the pre/post-pull messages.
 
         We only inject the messages if the client supports sideband, and the
         response has the format:
             0008NAK\n...0000
 
         Note that we do not check the no-progress capability as by default, git
         sends it, which effectively would block all messages.
         """
         if not self.SIDE_BAND_CAPS.intersection(capabilities):
             return response
 
         if not start_messages and not end_messages:
             return response
 
         # make a list out of response if it's an iterator
         # so we can investigate it for message injection.
         if hasattr(response, '__iter__'):
             response = list(response)
 
         if (not response[0].startswith('0008NAK\n') or
                 not response[-1].endswith('0000')):
             return response
 
         new_response = ['0008NAK\n']
         new_response.extend(self._get_messages(start_messages, capabilities))
         if len(response) == 1:
             new_response.append(response[0][8:-4])
         else:
             new_response.append(response[0][8:])
             new_response.extend(response[1:-1])
             new_response.append(response[-1][:-4])
         new_response.extend(self._get_messages(end_messages, capabilities))
         new_response.append('0000')
 
         return new_response
 
     def backend(self, request, environ):
         """
         WSGI Response producer for HTTP POST Git Smart HTTP requests.
         Reads commands and data from HTTP POST's body.
         returns an iterator obj with contents of git command's
         response to stdout
         """
         # TODO(skreft): think how we could detect an HTTPLockedException, as
         # we probably want to have the same mechanism used by mercurial and
         # simplevcs.
         # For that we would need to parse the output of the command looking for
         # some signs of the HTTPLockedError, parse the data and reraise it in
         # pygrack. However, that would interfere with the streaming.
         #
         # Now the output of a blocked push is:
         # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
         # POST git-receive-pack (1047 bytes)
         # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
         # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
         # ! [remote rejected] master -> master (pre-receive hook declined)
         # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
 
         git_command = self._get_fixedpath(request.path_info)
         if git_command not in self.commands:
             log.debug('command %s not allowed', git_command)
             return exc.HTTPForbidden()
 
         capabilities = None
         if git_command == 'git-upload-pack':
             capabilities = self._get_want_capabilities(request)
 
         if 'CONTENT_LENGTH' in environ:
             inputstream = FileWrapper(request.body_file_seekable,
                                       request.content_length)
         else:
             inputstream = request.body_file_seekable
 
         resp = Response()
         resp.content_type = ('application/x-%s-result' %
                              git_command.encode('utf8'))
         resp.charset = None
 
         pre_pull_messages = ''
         if git_command == 'git-upload-pack':
             status, pre_pull_messages = hooks.git_pre_pull(self.extras)
             if status != 0:
                 resp.app_iter = self._build_failed_pre_pull_response(
                     capabilities, pre_pull_messages)
                 return resp
 
         gitenv = dict(os.environ)
         # forget all configs
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
         cmd = [self.git_path, git_command[4:], '--stateless-rpc',
                self.content_path]
         log.debug('handling cmd %s', cmd)
 
         out = subprocessio.SubprocessIOChunker(
             cmd,
             inputstream=inputstream,
             env=gitenv,
             cwd=self.content_path,
             shell=False,
             fail_on_stderr=False,
             fail_on_return_code=False
         )
 
         if self.update_server_info and git_command == 'git-receive-pack':
             # We need to fully consume the iterator here, as the
             # update-server-info command needs to be run after the push.
             out = list(out)
 
             # Updating refs manually after each push.
             # This is required as some clients are exposing Git repos internally
             # with the dumb protocol.
             cmd = [self.git_path, 'update-server-info']
             log.debug('handling cmd %s', cmd)
             output = subprocessio.SubprocessIOChunker(
                 cmd,
                 inputstream=inputstream,
                 env=gitenv,
                 cwd=self.content_path,
                 shell=False,
                 fail_on_stderr=False,
                 fail_on_return_code=False
             )
             # Consume all the output so the subprocess finishes
             for _ in output:
                 pass
 
         if git_command == 'git-upload-pack':
             unused_status, post_pull_messages = hooks.git_post_pull(self.extras)
             resp.app_iter = self._inject_messages_to_response(
                 out, capabilities, pre_pull_messages, post_pull_messages)
         else:
             resp.app_iter = out
 
         return resp
 
     def __call__(self, environ, start_response):
         request = Request(environ)
         _path = self._get_fixedpath(request.path_info)
         if _path.startswith('info/refs'):
             app = self.inforefs
         else:
             app = self.backend
 
         try:
             resp = app(request, environ)
         except exc.HTTPException as error:
             log.exception('HTTP Error')
             resp = error
         except Exception:
             log.exception('Unknown error')
             resp = exc.HTTPInternalServerError()
 
         return resp(environ, start_response)

vcsserver/remote_wsgi.py

@@ -1,34 +1,34 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 from vcsserver import scm_app, wsgi_app_caller
 
 
 class GitRemoteWsgi(object):
     def handle(self, environ, input_data, *args, **kwargs):
         app = wsgi_app_caller.WSGIAppCaller(
             scm_app.create_git_wsgi_app(*args, **kwargs))
 
         return app.handle(environ, input_data)
 
 
 class HgRemoteWsgi(object):
     def handle(self, environ, input_data, *args, **kwargs):
         app = wsgi_app_caller.WSGIAppCaller(
             scm_app.create_hg_wsgi_app(*args, **kwargs))
 
         return app.handle(environ, input_data)

vcsserver/scm_app.py

@@ -1,235 +1,235 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import logging
 import itertools
 
 import mercurial
 import mercurial.error
 import mercurial.wireprotoserver
 import mercurial.hgweb.common
 import mercurial.hgweb.hgweb_mod
 import webob.exc
 
 from vcsserver import pygrack, exceptions, settings, git_lfs
 
 
 log = logging.getLogger(__name__)
 
 
 # propagated from mercurial documentation
 HG_UI_SECTIONS = [
     'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
     'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
     'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
 ]
 
 
 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
     """Extension of hgweb that simplifies some functions."""
 
     def _get_view(self, repo):
         """Views are not supported."""
         return repo
 
     def loadsubweb(self):
         """The result is only used in the templater method which is not used."""
         return None
 
     def run(self):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def templater(self, req):
         """Function used in an unreachable code path.
 
         This code is unreachable because we guarantee that the HTTP request,
         corresponds to a Mercurial command. See the is_hg method. So, we are
         never going to get a user-visible url.
         """
         raise NotImplementedError
 
     def archivelist(self, nodeid):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def __call__(self, environ, start_response):
         """Run the WSGI application.
 
         This may be called by multiple threads.
         """
         from mercurial.hgweb import request as requestmod
         req = requestmod.parserequestfromenv(environ)
         res = requestmod.wsgiresponse(req, start_response)
         gen = self.run_wsgi(req, res)
 
         first_chunk = None
 
         try:
             data = gen.next()
 
             def first_chunk():
                 yield data
         except StopIteration:
             pass
 
         if first_chunk:
             return itertools.chain(first_chunk(), gen)
         return gen
 
     def _runwsgi(self, req, res, repo):
 
         cmd = req.qsparams.get('cmd', '')
         if not mercurial.wireprotoserver.iscmd(cmd):
             # NOTE(marcink): for unsupported commands, we return bad request
             # internally from HG
             from mercurial.hgweb.common import statusmessage
             res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST)
             res.setbodybytes('')
             return res.sendresponse()
 
         return super(HgWeb, self)._runwsgi(req, res, repo)
 
 
 def make_hg_ui_from_config(repo_config):
     baseui = mercurial.ui.ui()
 
     # clean the baseui object
     baseui._ocfg = mercurial.config.config()
     baseui._ucfg = mercurial.config.config()
     baseui._tcfg = mercurial.config.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     return baseui
 
 
 def update_hg_ui_from_hgrc(baseui, repo_path):
     path = os.path.join(repo_path, '.hg', 'hgrc')
 
     if not os.path.isfile(path):
         log.debug('hgrc file is not present at %s, skipping...', path)
         return
     log.debug('reading hgrc from %s', path)
     cfg = mercurial.config.config()
     cfg.read(path)
     for section in HG_UI_SECTIONS:
         for k, v in cfg.items(section):
             log.debug('settings ui from file: [%s] %s=%s', section, k, v)
             baseui.setconfig(section, k, v)
 
 
 def create_hg_wsgi_app(repo_path, repo_name, config):
     """
     Prepares a WSGI application to handle Mercurial requests.
 
     :param config: is a list of 3-item tuples representing a ConfigObject
         (it is the serialized version of the config object).
     """
     log.debug("Creating Mercurial WSGI application")
 
     baseui = make_hg_ui_from_config(config)
     update_hg_ui_from_hgrc(baseui, repo_path)
 
     try:
         return HgWeb(repo_path, name=repo_name, baseui=baseui)
     except mercurial.error.RequirementError as e:
         raise exceptions.RequirementException(e)(e)
 
 
 class GitHandler(object):
     """
     Handler for Git operations like push/pull etc
     """
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def __call__(self, environ, start_response):
         app = webob.exc.HTTPNotFound()
         candidate_paths = (
             self.content_path, os.path.join(self.content_path, '.git'))
 
         for content_path in candidate_paths:
             try:
                 app = pygrack.GitRepository(
                     self.repo_name, content_path, self.git_path,
                     self.update_server_info, self.extras)
                 break
             except OSError:
                 continue
 
         return app(environ, start_response)
 
 
 def create_git_wsgi_app(repo_path, repo_name, config):
     """
     Creates a WSGI application to handle Git requests.
 
     :param config: is a dictionary holding the extras.
     """
     git_path = settings.GIT_EXECUTABLE
     update_server_info = config.pop('git_update_server_info')
     app = GitHandler(
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app
 
 
 class GitLFSHandler(object):
     """
     Handler for Git LFS operations
     """
 
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def get_app(self, git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
         app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)
         return app
 
 
 def create_git_lfs_wsgi_app(repo_path, repo_name, config):
     git_path = settings.GIT_EXECUTABLE
     update_server_info = config.pop('git_update_server_info')
     git_lfs_enabled = config.pop('git_lfs_enabled')
     git_lfs_store_path = config.pop('git_lfs_store_path')
     git_lfs_http_scheme = config.pop('git_lfs_http_scheme', 'http')
     app = GitLFSHandler(
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app.get_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme)

vcsserver/server.py

@@ -1,78 +1,78 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import gc
 import logging
 import os
 import time
 
 
 log = logging.getLogger(__name__)
 
 
 class VcsServer(object):
     """
     Exposed remote interface of the vcsserver itself.
 
     This object can be used to manage the server remotely. Right now the main
     use case is to allow to shut down the server.
     """
 
     _shutdown = False
 
     def shutdown(self):
         self._shutdown = True
 
     def ping(self):
         """
         Utility to probe a server connection.
         """
         log.debug("Received server ping.")
 
     def echo(self, data):
         """
         Utility for performance testing.
 
         Allows to pass in arbitrary data and will return this data.
         """
         log.debug("Received server echo.")
         return data
 
     def sleep(self, seconds):
         """
         Utility to simulate long running server interaction.
         """
         log.debug("Sleeping %s seconds", seconds)
         time.sleep(seconds)
 
     def get_pid(self):
         """
         Allows to discover the PID based on a proxy object.
         """
         return os.getpid()
 
     def run_gc(self):
         """
         Allows to trigger the garbage collector.
 
         Main intention is to support statistics gathering during test runs.
         """
         freed_objects = gc.collect()
         return {
             'freed_objects': freed_objects,
             'garbage': len(gc.garbage),
         }

vcsserver/settings.py

@@ -1,22 +1,22 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 WIRE_ENCODING = 'UTF-8'
 GIT_EXECUTABLE = 'git'
 SVN_EXECUTABLE = 'svn'
 SVNLOOK_EXECUTABLE = 'svnlook'
 BINARY_DIR = ''

vcsserver/svn.py

@@ -1,799 +1,791 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 from __future__ import absolute_import
 
 import os
 import subprocess
 from urllib2 import URLError
 import urlparse
 import logging
 import posixpath as vcspath
 import StringIO
 import urllib
 import traceback
 
 import svn.client
 import svn.core
 import svn.delta
 import svn.diff
 import svn.fs
 import svn.repos
 
 from vcsserver import svn_diff, exceptions, subprocessio, settings
 from vcsserver.base import RepoFactory, raise_from_original
 from vcsserver.vcs_base import RemoteBase
 
 log = logging.getLogger(__name__)
 
 
-# Set of svn compatible version flags.
-# Compare with subversion/svnadmin/svnadmin.c
-svn_compatible_versions = {
-    'pre-1.4-compatible',
-    'pre-1.5-compatible',
-    'pre-1.6-compatible',
-    'pre-1.8-compatible',
-    'pre-1.9-compatible'
-}
-
 svn_compatible_versions_map = {
     'pre-1.4-compatible': '1.3',
     'pre-1.5-compatible': '1.4',
     'pre-1.6-compatible': '1.5',
     'pre-1.8-compatible': '1.7',
     'pre-1.9-compatible': '1.8',
 }
 
+current_compatible_version = '1.12'
+
 
 def reraise_safe_exceptions(func):
     """Decorator for converting svn exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in svn remote call")
                 raise_from_original(exceptions.UnhandledException(e))
             raise
     return wrapper
 
 
 class SubversionFactory(RepoFactory):
     repo_type = 'svn'
 
     def _create_repo(self, wire, create, compatible_version):
         path = svn.core.svn_path_canonicalize(wire['path'])
         if create:
-            fs_config = {'compatible-version': '1.9'}
+            fs_config = {'compatible-version': current_compatible_version}
             if compatible_version:
-                if compatible_version not in svn_compatible_versions:
-                    raise Exception('Unknown SVN compatible version "{}"'
-                                    .format(compatible_version))
-                fs_config['compatible-version'] = \
-                    svn_compatible_versions_map[compatible_version]
+
+                compatible_version_string = \
+                    svn_compatible_versions_map.get(compatible_version) \
+                    or compatible_version
+                fs_config['compatible-version'] = compatible_version_string
 
             log.debug('Create SVN repo with config "%s"', fs_config)
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
 
         log.debug('Got SVN object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
         """
         Get a repository instance for the given path.
         """
         return self._create_repo(wire, create, compatible_version)
 
 
 NODE_TYPE_MAPPING = {
     svn.core.svn_node_file: 'file',
     svn.core.svn_node_dir: 'dir',
 }
 
 
 class SvnRemote(RemoteBase):
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
         # TODO: Remove once we do not use internal Mercurial objects anymore
         # for subversion
         self._hg_factory = hg_factory
 
     @reraise_safe_exceptions
     def discover_svn_version(self):
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
         return svn_ver
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
 
         try:
             return self.lookup(wire, -1) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
     def check_url(self, url, config_items):
         # this can throw exception if not installed, but we detect this
         from hgsubversion import svnrepo
 
         baseui = self._hg_factory._create_config(config_items)
         # uuid function get's only valid UUID from proper repo, else
         # throws exception
         try:
             svnrepo.svnremoterepo(baseui, url).svn.uuid
         except Exception:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
             raise URLError(
                 '"%s" is not a valid Subversion source url.' % (url, ))
         return True
 
     def is_path_valid_repository(self, wire, path):
 
         # NOTE(marcink):  short circuit the check for SVN repo
         # the repos.open might be expensive to check, but we have one cheap
         # pre condition that we can use, to check for 'format' file
 
         if not os.path.isfile(os.path.join(path, 'format')):
             return False
 
         try:
             svn.repos.open(path)
         except svn.core.SubversionException:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
             return False
         return True
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         cmd = ['svnadmin', 'info', repo_path]
         stdout, stderr = subprocessio.run_command(cmd)
         return stdout
 
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         start_rev = None
         end_rev = None
         if start_ts:
             start_ts_svn = apr_time_t(start_ts)
             start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
         else:
             start_rev = 1
         if end_ts:
             end_ts_svn = apr_time_t(end_ts)
             end_rev = svn.repos.dated_revision(repo, end_ts_svn)
         else:
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
     def revision_properties(self, wire, revision):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _revision_properties(_repo_id, _revision):
             repo = self._factory.repo(wire)
             fs_ptr = svn.repos.fs(repo)
             return svn.fs.revision_proplist(fs_ptr, revision)
         return _revision_properties(repo_id, revision)
 
     def revision_changes(self, wire, revision):
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         editor = svn.repos.ChangeCollector(fsobj, rev_root)
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         base_dir = ""
         send_deltas = False
         svn.repos.replay2(
             rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
             editor_ptr, editor_baton, None)
 
         added = []
         changed = []
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
         for path, change in editor.changes.iteritems():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
             if change.item_kind == svn.core.svn_node_dir:
                 continue
             if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
                                    svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(
                     "Action %s not supported on path %s" % (
                         change.action, path))
 
         changes = {
             'added': added,
             'changed': changed,
             'removed': removed,
         }
         return changes
 
     @reraise_safe_exceptions
     def node_history(self, wire, path, revision, limit):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
             cross_copies = False
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             rev_root = svn.fs.revision_root(fsobj, revision)
 
             history_revisions = []
             history = svn.fs.node_history(rev_root, path)
             history = svn.fs.history_prev(history, cross_copies)
             while history:
                 __, node_revision = svn.fs.history_location(history)
                 history_revisions.append(node_revision)
                 if limit and len(history_revisions) >= limit:
                     break
                 history = svn.fs.history_prev(history, cross_copies)
             return history_revisions
         return _assert_correct_path(context_uid, repo_id, path, revision, limit)
 
     def node_properties(self, wire, path, revision):
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _node_properties(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             rev_root = svn.fs.revision_root(fsobj, revision)
             return svn.fs.node_proplist(rev_root, path)
         return _node_properties(repo_id, path, revision)
 
     def file_annotate(self, wire, path, revision):
         abs_path = 'file://' + urllib.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
         start_rev = svn_opt_revision_value_t(0)
         peg_rev = svn_opt_revision_value_t(revision)
         end_rev = peg_rev
 
         annotations = []
 
         def receiver(line_no, revision, author, date, line, pool):
             annotations.append((line_no, revision, line))
 
         # TODO: Cannot use blame5, missing typemap function in the swig code
         try:
             svn.client.blame2(
                 file_uri, peg_rev, start_rev, end_rev,
                 receiver, svn.client.create_context())
         except svn.core.SubversionException as exc:
             log.exception("Error during blame operation.")
             raise Exception(
                 "Blame not supported or file does not exist at path %s. "
                 "Error %s." % (path, exc))
 
         return annotations
 
     def get_node_type(self, wire, path, revision=None):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_node_type(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fs_ptr = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_rev(fs_ptr)
             root = svn.fs.revision_root(fs_ptr, _revision)
             node = svn.fs.check_path(root, path)
             return NODE_TYPE_MAPPING.get(node, None)
         return _get_node_type(repo_id, path, revision)
 
     def get_nodes(self, wire, path, revision=None):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_nodes(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_rev(fsobj)
             root = svn.fs.revision_root(fsobj, _revision)
             entries = svn.fs.dir_entries(root, path)
             result = []
             for entry_path, entry_info in entries.iteritems():
                 result.append(
                     (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
             return result
         return _get_nodes(repo_id, path, revision)
 
     def get_file_content(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
         return content.read()
 
     def get_file_size(self, wire, path, revision=None):
 
         cache_on, context_uid, repo_id = self._cache_on(wire)
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _get_file_size(_repo_id, _path, _revision):
             repo = self._factory.repo(wire)
             fsobj = svn.repos.fs(repo)
             if _revision is None:
                 _revision = svn.fs.youngest_revision(fsobj)
             root = svn.fs.revision_root(fsobj, _revision)
             size = svn.fs.file_length(root, path)
             return size
         return _get_file_size(repo_id, path, revision)
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
     def get_url_and_credentials(self, src_url):
         obj = urlparse.urlparse(src_url)
         username = obj.username or None
         password = obj.password or None
         return username, password, src_url
 
     def import_remote_repository(self, wire, src_url):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         username, password, src_url = self.get_url_and_credentials(src_url)
         rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
                      '--trust-server-cert-failures=unknown-ca']
         if username and password:
             rdump_cmd += ['--username', username, '--password', password]
         rdump_cmd += [src_url]
 
         rdump = subprocess.Popen(
             rdump_cmd,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         load = subprocess.Popen(
             ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
 
         # TODO: johbo: This can be a very long operation, might be better
         # to track some kind of status and provide an api to check if the
         # import is done.
         rdump.wait()
         load.wait()
 
         log.debug('Return process ended with code: %s', rdump.returncode)
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
             log.error('svnrdump dump failed: statuscode %s: message: %s',
                       rdump.returncode, errors)
             reason = 'UNKNOWN'
             if 'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
 
             if reason == 'UNKNOWN':
                 reason = 'UNKNOWN:{}'.format(errors)
             raise Exception(
                 'Failed to dump the remote repository from %s. Reason:%s' % (
                     src_url, reason))
         if load.returncode != 0:
             raise Exception(
                 'Failed to load the dump of remote repository from %s.' %
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
         assert isinstance(message, str)
         assert isinstance(author, str)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
 
         rev = svn.fs.youngest_rev(fsobj)
         txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
         txn_root = svn.fs.txn_root(txn)
 
         for node in updated:
             TxnNodeProcessor(node, txn_root).update()
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp:
             apr_time = apr_time_t(timestamp)
             ts_formatted = svn.core.svn_time_to_cstring(apr_time)
             svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
 
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
         wire.update(cache=False)
         repo = self._factory.repo(wire)
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
             return diff_creator.generate_diff()
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
                 "Path might not exist %s, %s" % (path1, path2))
             return ""
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return False
 
     @reraise_safe_exceptions
     def is_binary(self, wire, rev, path):
         cache_on, context_uid, repo_id = self._cache_on(wire)
 
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _is_binary(_repo_id, _rev, _path):
             raw_bytes = self.get_file_content(wire, path, rev)
             return raw_bytes and '\0' in raw_bytes
 
         return _is_binary(repo_id, rev, path)
 
     @reraise_safe_exceptions
     def run_svn_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         safe_call = False
         if '_safe' in opts:
             safe_call = True
 
         svnenv = os.environ.copy()
         svnenv.update(opts.pop('extra_env', {}))
 
         _opts = {'env': svnenv, 'shell': False}
 
         try:
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(p), ''.join(p.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run svn command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_svn_hooks
         repo_path = wire['path']
         binary_dir = settings.BINARY_DIR
         executable = None
         if binary_dir:
             executable = os.path.join(binary_dir, 'python')
         return install_svn_hooks(
             repo_path, executable=executable, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_svn_pre_hook_version, get_svn_post_hook_version)
         repo_path = wire['path']
         return {
             'pre_version': get_svn_pre_hook_version(repo_path),
             'post_version': get_svn_post_hook_version(repo_path),
         }
 
 
 class SvnDiffer(object):
     """
     Utility to create diffs based on difflib and the Subversion api
     """
 
     binary_content = False
 
     def __init__(
             self, repo, src_rev, src_path, tgt_rev, tgt_path,
             ignore_whitespace, context):
         self.repo = repo
         self.ignore_whitespace = ignore_whitespace
         self.context = context
 
         fsobj = svn.repos.fs(repo)
 
         self.tgt_rev = tgt_rev
         self.tgt_path = tgt_path or ''
         self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
         self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
 
         self.src_rev = src_rev
         self.src_path = src_path or self.tgt_path
         self.src_root = svn.fs.revision_root(fsobj, src_rev)
         self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
 
         self._validate()
 
     def _validate(self):
         if (self.tgt_kind != svn.core.svn_node_none and
                 self.src_kind != svn.core.svn_node_none and
                 self.src_kind != self.tgt_kind):
             # TODO: johbo: proper error handling
             raise Exception(
                 "Source and target are not compatible for diff generation. "
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
     def generate_diff(self):
         buf = StringIO.StringIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
     def _generate_dir_diff(self, buf):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
             self.src_root,
             self.src_path,
             '',  # src_entry
             self.tgt_root,
             self.tgt_path,
             editor_ptr, editor_baton,
             authorization_callback_allow_all,
             False,  # text_deltas
             svn.core.svn_depth_infinity,  # depth
             False,  # entry_props
             False,  # ignore_ancestry
         )
 
         for path, __, change in sorted(editor.changes):
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
     def _generate_file_diff(self, buf):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
         elif self.tgt_kind == svn.core.svn_node_none:
             change = "delete"
         tgt_base, tgt_path = vcspath.split(self.tgt_path)
         src_base, src_path = vcspath.split(self.src_path)
         self._generate_node_diff(
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
             self, buf, change, tgt_path, tgt_base, src_path, src_base):
 
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
             # we compare same revisions
             return
 
         tgt_full_path = vcspath.join(tgt_base, tgt_path)
         src_full_path = vcspath.join(src_base, src_path)
 
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
         if mime_type and not mime_type.startswith('text'):
             self.binary_content = True
             buf.write("=" * 67 + '\n')
             buf.write("Cannot display: file marked as a binary type.\n")
             buf.write("svn:mime-type = %s\n" % mime_type)
         buf.write("Index: %s\n" % (tgt_path, ))
         buf.write("=" * 67 + '\n')
         buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
             'tgt_path': tgt_path})
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
             buf.write("new file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- /dev/null\t(revision 0)\n")
             src_lines = []
         else:
             if change == 'delete':
                 buf.write("deleted file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- a/%s\t(revision %s)\n" % (
                 src_path, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
             buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
             tgt_lines = []
         else:
             buf.write("+++ b/%s\t(revision %s)\n" % (
                 tgt_path, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
             buf.writelines(udiff)
 
     def _get_mime_type(self, path):
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
         except svn.core.SubversionException:
             mime_type = svn.fs.node_prop(
                 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
         return mime_type
 
     def _svn_readlines(self, fs_root, node_path):
         if self.binary_content:
             return []
         node_kind = svn.fs.check_path(fs_root, node_path)
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
         content = svn.core.Stream(
             svn.fs.file_contents(fs_root, node_path)).read()
         return content.splitlines(True)
 
 
 class DiffChangeEditor(svn.delta.Editor):
     """
     Records changes between two given revisions
     """
 
     def __init__(self):
         self.changes = []
 
     def delete_entry(self, path, revision, parent_baton, pool=None):
         self.changes.append((path, None, 'delete'))
 
     def add_file(
             self, path, parent_baton, copyfrom_path, copyfrom_revision,
             file_pool=None):
         self.changes.append((path, 'file', 'add'))
 
     def open_file(self, path, parent_baton, base_revision, file_pool=None):
         self.changes.append((path, 'file', 'change'))
 
 
 def authorization_callback_allow_all(root, path, pool):
     return True
 
 
 class TxnNodeProcessor(object):
     """
     Utility to process the change of one node within a transaction root.
 
     It encapsulates the knowledge of how to add, update or remove
     a node for a given transaction root. The purpose is to support the method
     `SvnRemote.commit`.
     """
 
     def __init__(self, node, txn_root):
         assert isinstance(node['path'], str)
 
         self.node = node
         self.txn_root = txn_root
 
     def update(self):
         self._ensure_parent_dirs()
         self._add_file_if_node_does_not_exist()
         self._update_file_content()
         self._update_file_properties()
 
     def remove(self):
         svn.fs.delete(self.txn_root, self.node['path'])
         # TODO: Clean up directory if empty
 
     def _ensure_parent_dirs(self):
         curdir = vcspath.dirname(self.node['path'])
         dirs_to_create = []
         while not self._svn_path_exists(curdir):
             dirs_to_create.append(curdir)
             curdir = vcspath.dirname(curdir)
 
         for curdir in reversed(dirs_to_create):
             log.debug('Creating missing directory "%s"', curdir)
             svn.fs.make_dir(self.txn_root, curdir)
 
     def _svn_path_exists(self, path):
         path_status = svn.fs.check_path(self.txn_root, path)
         return path_status != svn.core.svn_node_none
 
     def _add_file_if_node_does_not_exist(self):
         kind = svn.fs.check_path(self.txn_root, self.node['path'])
         if kind == svn.core.svn_node_none:
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
         assert isinstance(self.node['content'], str)
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
         for key, value in properties.iteritems():
             svn.fs.change_node_prop(
                 self.txn_root, self.node['path'], key, value)
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
     return timestamp * 1E6
 
 
 def svn_opt_revision_value_t(num):
     """
     Put `num` into a `svn_opt_revision_value_t` structure.
     """
     value = svn.core.svn_opt_revision_value_t()
     value.number = num
     revision = svn.core.svn_opt_revision_t()
     revision.kind = svn.core.svn_opt_revision_number
     revision.value = value
     return revision

vcsserver/tests/__init__.py

@@ -1,16 +1,16 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/tests/conftest.py

@@ -1,57 +1,57 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import socket
 
 import pytest
 
 
 def pytest_addoption(parser):
     parser.addoption(
         '--repeat', type=int, default=100,
         help="Number of repetitions in performance tests.")
 
 
 @pytest.fixture(scope='session')
 def repeat(request):
     """
     The number of repetitions is based on this fixture.
 
     Slower calls may divide it by 10 or 100. It is chosen in a way so that the
     tests are not too slow in our default test suite.
     """
     return request.config.getoption('--repeat')
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = get_available_port()
     print('Using vcsserver port %s' % (port, ))
     return port
 
 
 def get_available_port():
     family = socket.AF_INET
     socktype = socket.SOCK_STREAM
     host = '127.0.0.1'
 
     mysocket = socket.socket(family, socktype)
     mysocket.bind((host, 0))
     port = mysocket.getsockname()[1]
     mysocket.close()
     del mysocket
     return port

vcsserver/tests/fixture.py

@@ -1,86 +1,86 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import shutil
 import tempfile
 
 import configobj
 
 
 class ContextINI(object):
     """
     Allows to create a new test.ini file as a copy of existing one with edited
     data. If existing file is not present, it creates a new one. Example usage::
 
         with TestINI('test.ini', [{'section': {'key': 'val'}}]) as new_test_ini_path:
             print 'vcsserver --config=%s' % new_test_ini
     """
 
     def __init__(self, ini_file_path, ini_params, new_file_prefix=None,
                  destroy=True):
         self.ini_file_path = ini_file_path
         self.ini_params = ini_params
         self.new_path = None
         self.new_path_prefix = new_file_prefix or 'test'
         self.destroy = destroy
 
     def __enter__(self):
         _, pref = tempfile.mkstemp()
         loc = tempfile.gettempdir()
         self.new_path = os.path.join(loc, '{}_{}_{}'.format(
             pref, self.new_path_prefix, self.ini_file_path))
 
         # copy ini file and modify according to the params, if we re-use a file
         if os.path.isfile(self.ini_file_path):
             shutil.copy(self.ini_file_path, self.new_path)
         else:
             # create new dump file for configObj to write to.
             with open(self.new_path, 'wb'):
                 pass
 
         config = configobj.ConfigObj(
             self.new_path, file_error=True, write_empty_values=True)
 
         for data in self.ini_params:
             section, ini_params = data.items()[0]
             key, val = ini_params.items()[0]
             if section not in config:
                 config[section] = {}
             config[section][key] = val
 
         config.write()
         return self.new_path
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.destroy:
             os.remove(self.new_path)
 
 
 def no_newline_id_generator(test_name):
     """
     Generates a test name without spaces or newlines characters. Used for
     nicer output of progress of test
     """
     org_name = test_name
     test_name = str(test_name)\
         .replace('\n', '_N') \
         .replace('\r', '_N') \
         .replace('\t', '_T') \
         .replace(' ', '_S')
 
     return test_name or 'test-with-empty-name'

vcsserver/tests/test_git.py

@@ -1,160 +1,160 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import inspect
 
 import pytest
 import dulwich.errors
 from mock import Mock, patch
 
 from vcsserver import git
 
 
 SAMPLE_REFS = {
     'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
     'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
     'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
     'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
     'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
 }
 
 
 @pytest.fixture
 def git_remote():
     """
     A GitRemote instance with a mock factory.
     """
     factory = Mock()
     remote = git.GitRemote(factory)
     return remote
 
 
 def test_discover_git_version(git_remote):
     version = git_remote.discover_git_version()
     assert version
 
 
 class TestGitFetch(object):
     def setup(self):
         self.mock_repo = Mock()
         factory = Mock()
         factory.repo = Mock(return_value=self.mock_repo)
         self.remote_git = git.GitRemote(factory)
 
     def test_fetches_all_when_no_commit_ids_specified(self):
         def side_effect(determine_wants, *args, **kwargs):
             determine_wants(SAMPLE_REFS)
 
         with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
             mock_fetch.side_effect = side_effect
             self.remote_git.pull(wire={}, url='/tmp/', apply_refs=False)
             determine_wants = self.mock_repo.object_store.determine_wants_all
             determine_wants.assert_called_once_with(SAMPLE_REFS)
 
     def test_fetches_specified_commits(self):
         selected_refs = {
             'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
             'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         def side_effect(determine_wants, *args, **kwargs):
             result = determine_wants(SAMPLE_REFS)
             assert sorted(result) == sorted(selected_refs.values())
             return result
 
         with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
             mock_fetch.side_effect = side_effect
             self.remote_git.pull(
                 wire={}, url='/tmp/', apply_refs=False,
                 refs=selected_refs.keys())
             determine_wants = self.mock_repo.object_store.determine_wants_all
             assert determine_wants.call_count == 0
 
     def test_get_remote_refs(self):
         factory = Mock()
         remote_git = git.GitRemote(factory)
         url = 'http://example.com/test/test.git'
         sample_refs = {
             'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
             'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         with patch('vcsserver.git.Repo', create=False) as mock_repo:
             mock_repo().get_refs.return_value = sample_refs
             remote_refs = remote_git.get_remote_refs(wire={}, url=url)
             mock_repo().get_refs.assert_called_once_with()
             assert remote_refs == sample_refs
 
 
 class TestReraiseSafeExceptions(object):
 
     def test_method_decorated_with_reraise_safe_exceptions(self):
         factory = Mock()
         git_remote = git.GitRemote(factory)
 
         def fake_function():
             return None
 
         decorator = git.reraise_safe_exceptions(fake_function)
 
         methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
         for method_name, method in methods:
             if not method_name.startswith('_'):
                 assert method.im_func.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
         (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
         (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
         (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
         (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
         (dulwich.errors.HangupException(), 'error'),
         (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
     ])
     def test_safe_exceptions_reraised(self, side_effect, expected_type):
         @git.reraise_safe_exceptions
         def fake_method():
             raise side_effect
 
         with pytest.raises(Exception) as exc_info:
             fake_method()
         assert type(exc_info.value) == Exception
         assert exc_info.value._vcs_kind == expected_type
 
 
 class TestDulwichRepoWrapper(object):
     def test_calls_close_on_delete(self):
         isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
         with isdir_patcher:
             repo = git.Repo('/tmp/abcde')
         with patch.object(git.DulwichRepo, 'close') as close_mock:
             del repo
         close_mock.assert_called_once_with()
 
 
 class TestGitFactory(object):
     def test_create_repo_returns_dulwich_wrapper(self):
 
         with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
             mock.side_effect = {'repo_objects': ''}
             factory = git.GitFactory()
             wire = {
                 'path': '/tmp/abcde'
             }
             isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
             with isdir_patcher:
                 result = factory._create_repo(wire, True)
             assert isinstance(result, git.Repo)

vcsserver/tests/test_hg.py

@@ -1,108 +1,108 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import inspect
 import sys
 import traceback
 
 import pytest
 from mercurial.error import LookupError
 from mock import Mock, MagicMock, patch
 
 from vcsserver import exceptions, hg, hgcompat
 
 
 class TestDiff(object):
     def test_raising_safe_exception_when_lookup_failed(self):
 
         factory = Mock()
         hg_remote = hg.HgRemote(factory)
         with patch('mercurial.patch.diff') as diff_mock:
             diff_mock.side_effect = LookupError(
                 'deadbeef', 'index', 'message')
             with pytest.raises(Exception) as exc_info:
                 hg_remote.diff(
                     wire={}, commit_id_1='deadbeef', commit_id_2='deadbee1',
                     file_filter=None, opt_git=True, opt_ignorews=True,
                     context=3)
             assert type(exc_info.value) == Exception
             assert exc_info.value._vcs_kind == 'lookup'
 
 
 class TestReraiseSafeExceptions(object):
     def test_method_decorated_with_reraise_safe_exceptions(self):
         factory = Mock()
         hg_remote = hg.HgRemote(factory)
         methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
         decorator = hg.reraise_safe_exceptions(None)
         for method_name, method in methods:
             if not method_name.startswith('_'):
                 assert method.im_func.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
         (hgcompat.Abort(), 'abort'),
         (hgcompat.InterventionRequired(), 'abort'),
         (hgcompat.RepoLookupError(), 'lookup'),
         (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
         (hgcompat.RepoError(), 'error'),
         (hgcompat.RequirementError(), 'requirement'),
     ])
     def test_safe_exceptions_reraised(self, side_effect, expected_type):
         @hg.reraise_safe_exceptions
         def fake_method():
             raise side_effect
 
         with pytest.raises(Exception) as exc_info:
             fake_method()
         assert type(exc_info.value) == Exception
         assert exc_info.value._vcs_kind == expected_type
 
     def test_keeps_original_traceback(self):
         @hg.reraise_safe_exceptions
         def fake_method():
             try:
                 raise hgcompat.Abort()
             except:
                 self.original_traceback = traceback.format_tb(
                     sys.exc_info()[2])
                 raise
 
         try:
             fake_method()
         except Exception:
             new_traceback = traceback.format_tb(sys.exc_info()[2])
 
         new_traceback_tail = new_traceback[-len(self.original_traceback):]
         assert new_traceback_tail == self.original_traceback
 
     def test_maps_unknow_exceptions_to_unhandled(self):
         @hg.reraise_safe_exceptions
         def stub_method():
             raise ValueError('stub')
 
         with pytest.raises(Exception) as exc_info:
             stub_method()
         assert exc_info.value._vcs_kind == 'unhandled'
 
     def test_does_not_map_known_exceptions(self):
         @hg.reraise_safe_exceptions
         def stub_method():
             raise exceptions.LookupException()('stub')
 
         with pytest.raises(Exception) as exc_info:
             stub_method()
         assert exc_info.value._vcs_kind == 'lookup'

vcsserver/tests/test_hgpatches.py

@@ -1,124 +1,124 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import mock
 import pytest
 
 from vcsserver import hgcompat, hgpatches
 
 
 LARGEFILES_CAPABILITY = 'largefiles=serve'
 
 
 def test_patch_largefiles_capabilities_applies_patch(
         patched_capabilities):
     lfproto = hgcompat.largefiles.proto
     hgpatches.patch_largefiles_capabilities()
     assert lfproto._capabilities.func_name == '_dynamic_capabilities'
 
 
 def test_dynamic_capabilities_uses_original_function_if_not_enabled(
         stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities,
         orig_capabilities):
     dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
         hgcompat.largefiles.proto, stub_extensions)
 
     caps = dynamic_capabilities(orig_capabilities, stub_repo, stub_proto)
 
     stub_extensions.assert_called_once_with(stub_ui)
     assert LARGEFILES_CAPABILITY not in caps
 
 
 def test_dynamic_capabilities_ignores_updated_capabilities(
         stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities,
         orig_capabilities):
     stub_extensions.return_value = [('largefiles', mock.Mock())]
     dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
         hgcompat.largefiles.proto, stub_extensions)
 
     # This happens when the extension is loaded for the first time, important
     # to ensure that an updated function is correctly picked up.
     hgcompat.largefiles.proto._capabilities = mock.Mock(
         side_effect=Exception('Must not be called'))
 
     dynamic_capabilities(orig_capabilities, stub_repo, stub_proto)
 
 
 def test_dynamic_capabilities_uses_largefiles_if_enabled(
         stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities,
         orig_capabilities):
     stub_extensions.return_value = [('largefiles', mock.Mock())]
 
     dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
         hgcompat.largefiles.proto, stub_extensions)
 
     caps = dynamic_capabilities(orig_capabilities, stub_repo, stub_proto)
 
     stub_extensions.assert_called_once_with(stub_ui)
     assert LARGEFILES_CAPABILITY in caps
 
 
 def test_hgsubversion_import():
     from hgsubversion import svnrepo
     assert svnrepo
 
 
 @pytest.fixture
 def patched_capabilities(request):
     """
     Patch in `capabilitiesorig` and restore both capability functions.
     """
     lfproto = hgcompat.largefiles.proto
     orig_capabilities = lfproto._capabilities
 
     @request.addfinalizer
     def restore():
         lfproto._capabilities = orig_capabilities
 
 
 @pytest.fixture
 def stub_repo(stub_ui):
     repo = mock.Mock()
     repo.ui = stub_ui
     return repo
 
 
 @pytest.fixture
 def stub_proto(stub_ui):
     proto = mock.Mock()
     proto.ui = stub_ui
     return proto
 
 
 @pytest.fixture
 def orig_capabilities():
     from mercurial.wireprotov1server import wireprotocaps
 
     def _capabilities(repo, proto):
         return wireprotocaps
     return _capabilities
 
 
 @pytest.fixture
 def stub_ui():
     return hgcompat.ui.ui()
 
 
 @pytest.fixture
 def stub_extensions():
     extensions = mock.Mock(return_value=tuple())
     return extensions

vcsserver/tests/test_hooks.py

@@ -1,241 +1,241 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import contextlib
 import io
 import threading
 from BaseHTTPServer import BaseHTTPRequestHandler
 from SocketServer import TCPServer
 
 import mercurial.ui
 import mock
 import pytest
 import simplejson as json
 
 from vcsserver import hooks
 
 
 def get_hg_ui(extras=None):
     """Create a Config object with a valid RC_SCM_DATA entry."""
     extras = extras or {}
     required_extras = {
         'username': '',
         'repository': '',
         'locked_by': '',
         'scm': '',
         'make_lock': '',
         'action': '',
         'ip': '',
         'hooks_uri': 'fake_hooks_uri',
     }
     required_extras.update(extras)
     hg_ui = mercurial.ui.ui()
     hg_ui.setconfig('rhodecode', 'RC_SCM_DATA', json.dumps(required_extras))
 
     return hg_ui
 
 
 def test_git_pre_receive_is_disabled():
     extras = {'hooks': ['pull']}
     response = hooks.git_pre_receive(None, None,
                                      {'RC_SCM_DATA': json.dumps(extras)})
 
     assert response == 0
 
 
 def test_git_post_receive_is_disabled():
     extras = {'hooks': ['pull']}
     response = hooks.git_post_receive(None, '',
                                       {'RC_SCM_DATA': json.dumps(extras)})
 
     assert response == 0
 
 
 def test_git_post_receive_calls_repo_size():
     extras = {'hooks': ['push', 'repo_size']}
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     extras.update({'commit_ids': [], 'hook_type': 'post_receive',
                    'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
     expected_calls = [
         mock.call('repo_size', extras, mock.ANY),
         mock.call('post_push', extras, mock.ANY),
     ]
     assert call_hook_mock.call_args_list == expected_calls
 
 
 def test_git_post_receive_does_not_call_disabled_repo_size():
     extras = {'hooks': ['push']}
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     extras.update({'commit_ids': [], 'hook_type': 'post_receive',
                    'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}})
     expected_calls = [
         mock.call('post_push', extras, mock.ANY)
     ]
     assert call_hook_mock.call_args_list == expected_calls
 
 
 def test_repo_size_exception_does_not_affect_git_post_receive():
     extras = {'hooks': ['push', 'repo_size']}
     status = 0
 
     def side_effect(name, *args, **kwargs):
         if name == 'repo_size':
             raise Exception('Fake exception')
         else:
             return status
 
     with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
         call_hook_mock.side_effect = side_effect
         result = hooks.git_post_receive(
             None, '', {'RC_SCM_DATA': json.dumps(extras)})
     assert result == status
 
 
 def test_git_pre_pull_is_disabled():
     assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
 
 
 def test_git_post_pull_is_disabled():
     assert (
         hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
 
 
 class TestGetHooksClient(object):
 
     def test_returns_http_client_when_protocol_matches(self):
         hooks_uri = 'localhost:8000'
         result = hooks._get_hooks_client({
             'hooks_uri': hooks_uri,
             'hooks_protocol': 'http'
         })
         assert isinstance(result, hooks.HooksHttpClient)
         assert result.hooks_uri == hooks_uri
 
     def test_returns_dummy_client_when_hooks_uri_not_specified(self):
         fake_module = mock.Mock()
         import_patcher = mock.patch.object(
             hooks.importlib, 'import_module', return_value=fake_module)
         fake_module_name = 'fake.module'
         with import_patcher as import_mock:
             result = hooks._get_hooks_client(
                 {'hooks_module': fake_module_name})
 
         import_mock.assert_called_once_with(fake_module_name)
         assert isinstance(result, hooks.HooksDummyClient)
         assert result._hooks_module == fake_module
 
 
 class TestHooksHttpClient(object):
     def test_init_sets_hooks_uri(self):
         uri = 'localhost:3000'
         client = hooks.HooksHttpClient(uri)
         assert client.hooks_uri == uri
 
     def test_serialize_returns_json_string(self):
         client = hooks.HooksHttpClient('localhost:3000')
         hook_name = 'test'
         extras = {
             'first': 1,
             'second': 'two'
         }
         result = client._serialize(hook_name, extras)
         expected_result = json.dumps({
             'method': hook_name,
             'extras': extras
         })
         assert result == expected_result
 
     def test_call_queries_http_server(self, http_mirror):
         client = hooks.HooksHttpClient(http_mirror.uri)
         hook_name = 'test'
         extras = {
             'first': 1,
             'second': 'two'
         }
         result = client(hook_name, extras)
         expected_result = {
             'method': hook_name,
             'extras': extras
         }
         assert result == expected_result
 
 
 class TestHooksDummyClient(object):
     def test_init_imports_hooks_module(self):
         hooks_module_name = 'rhodecode.fake.module'
         hooks_module = mock.MagicMock()
 
         import_patcher = mock.patch.object(
             hooks.importlib, 'import_module', return_value=hooks_module)
         with import_patcher as import_mock:
             client = hooks.HooksDummyClient(hooks_module_name)
             import_mock.assert_called_once_with(hooks_module_name)
         assert client._hooks_module == hooks_module
 
     def test_call_returns_hook_result(self):
         hooks_module_name = 'rhodecode.fake.module'
         hooks_module = mock.MagicMock()
         import_patcher = mock.patch.object(
             hooks.importlib, 'import_module', return_value=hooks_module)
         with import_patcher:
             client = hooks.HooksDummyClient(hooks_module_name)
 
         result = client('post_push', {})
         hooks_module.Hooks.assert_called_once_with()
         assert result == hooks_module.Hooks().__enter__().post_push()
 
 
 @pytest.fixture
 def http_mirror(request):
     server = MirrorHttpServer()
     request.addfinalizer(server.stop)
     return server
 
 
 class MirrorHttpHandler(BaseHTTPRequestHandler):
     def do_POST(self):
         length = int(self.headers['Content-Length'])
         body = self.rfile.read(length).decode('utf-8')
         self.send_response(200)
         self.end_headers()
         self.wfile.write(body)
 
 
 class MirrorHttpServer(object):
     ip_address = '127.0.0.1'
     port = 0
 
     def __init__(self):
         self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
         _, self.port = self._daemon.server_address
         self._thread = threading.Thread(target=self._daemon.serve_forever)
         self._thread.daemon = True
         self._thread.start()
 
     def stop(self):
         self._daemon.shutdown()
         self._thread.join()
         self._daemon = None
         self._thread = None
 
     @property
     def uri(self):
         return '{}:{}'.format(self.ip_address, self.port)

vcsserver/tests/test_install_hooks.py

@@ -1,206 +1,206 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import sys
 import stat
 import pytest
 import vcsserver
 import tempfile
 from vcsserver import hook_utils
 from vcsserver.tests.fixture import no_newline_id_generator
 from vcsserver.utils import AttributeDict
 
 
 class TestCheckRhodecodeHook(object):
 
     def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir):
         hook = os.path.join(str(tmpdir), 'fake_hook_file.py')
         with open(hook, 'wb') as f:
             f.write('dummy test')
             result = hook_utils.check_rhodecode_hook(hook)
             assert result is False
 
     def test_returns_true_when_no_hook_file_found(self, tmpdir):
         hook = os.path.join(str(tmpdir), 'fake_hook_file_not_existing.py')
         result = hook_utils.check_rhodecode_hook(hook)
         assert result
 
     @pytest.mark.parametrize("file_content, expected_result", [
         ("RC_HOOK_VER = '3.3.3'\n", True),
         ("RC_HOOK = '3.3.3'\n", False),
     ], ids=no_newline_id_generator)
     def test_signatures(self, file_content, expected_result, tmpdir):
         hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py')
         with open(hook, 'wb') as f:
             f.write(file_content)
 
         result = hook_utils.check_rhodecode_hook(hook)
 
         assert result is expected_result
 
 
 class BaseInstallHooks(object):
     HOOK_FILES = ()
 
     def _check_hook_file_mode(self, file_path):
         assert os.path.exists(file_path), 'path %s missing' % file_path
         stat_info = os.stat(file_path)
 
         file_mode = stat.S_IMODE(stat_info.st_mode)
         expected_mode = int('755', 8)
         assert expected_mode == file_mode
 
     def _check_hook_file_content(self, file_path, executable):
         executable = executable or sys.executable
         with open(file_path, 'rt') as hook_file:
             content = hook_file.read()
 
         expected_env = '#!{}'.format(executable)
         expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(
             vcsserver.__version__)
         assert content.strip().startswith(expected_env)
         assert expected_rc_version in content
 
     def _create_fake_hook(self, file_path, content):
         with open(file_path, 'w') as hook_file:
             hook_file.write(content)
 
     def create_dummy_repo(self, repo_type):
         tmpdir = tempfile.mkdtemp()
         repo = AttributeDict()
         if repo_type == 'git':
             repo.path = os.path.join(tmpdir, 'test_git_hooks_installation_repo')
             os.makedirs(repo.path)
             os.makedirs(os.path.join(repo.path, 'hooks'))
             repo.bare = True
 
         elif repo_type == 'svn':
             repo.path = os.path.join(tmpdir, 'test_svn_hooks_installation_repo')
             os.makedirs(repo.path)
             os.makedirs(os.path.join(repo.path, 'hooks'))
 
         return repo
 
     def check_hooks(self, repo_path, repo_bare=True):
         for file_name in self.HOOK_FILES:
             if repo_bare:
                 file_path = os.path.join(repo_path, 'hooks', file_name)
             else:
                 file_path = os.path.join(repo_path, '.git', 'hooks', file_name)
             self._check_hook_file_mode(file_path)
             self._check_hook_file_content(file_path, sys.executable)
 
 
 class TestInstallGitHooks(BaseInstallHooks):
     HOOK_FILES = ('pre-receive', 'post-receive')
 
     def test_hooks_are_installed(self):
         repo = self.create_dummy_repo('git')
         result = hook_utils.install_git_hooks(repo.path, repo.bare)
         assert result
         self.check_hooks(repo.path, repo.bare)
 
     def test_hooks_are_replaced(self):
         repo = self.create_dummy_repo('git')
         hooks_path = os.path.join(repo.path, 'hooks')
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content="RC_HOOK_VER = 'abcde'\n")
 
         result = hook_utils.install_git_hooks(repo.path, repo.bare)
         assert result
         self.check_hooks(repo.path, repo.bare)
 
     def test_non_rc_hooks_are_not_replaced(self):
         repo = self.create_dummy_repo('git')
         hooks_path = os.path.join(repo.path, 'hooks')
         non_rc_content = 'echo "non rc hook"\n'
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content=non_rc_content)
 
         result = hook_utils.install_git_hooks(repo.path, repo.bare)
         assert result
 
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             with open(file_path, 'rt') as hook_file:
                 content = hook_file.read()
             assert content == non_rc_content
 
     def test_non_rc_hooks_are_replaced_with_force_flag(self):
         repo = self.create_dummy_repo('git')
         hooks_path = os.path.join(repo.path, 'hooks')
         non_rc_content = 'echo "non rc hook"\n'
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content=non_rc_content)
 
         result = hook_utils.install_git_hooks(
             repo.path, repo.bare, force_create=True)
         assert result
         self.check_hooks(repo.path, repo.bare)
 
 
 class TestInstallSvnHooks(BaseInstallHooks):
     HOOK_FILES = ('pre-commit', 'post-commit')
 
     def test_hooks_are_installed(self):
         repo = self.create_dummy_repo('svn')
         result = hook_utils.install_svn_hooks(repo.path)
         assert result
         self.check_hooks(repo.path)
 
     def test_hooks_are_replaced(self):
         repo = self.create_dummy_repo('svn')
         hooks_path = os.path.join(repo.path, 'hooks')
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content="RC_HOOK_VER = 'abcde'\n")
 
         result = hook_utils.install_svn_hooks(repo.path)
         assert result
         self.check_hooks(repo.path)
 
     def test_non_rc_hooks_are_not_replaced(self):
         repo = self.create_dummy_repo('svn')
         hooks_path = os.path.join(repo.path, 'hooks')
         non_rc_content = 'echo "non rc hook"\n'
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content=non_rc_content)
 
         result = hook_utils.install_svn_hooks(repo.path)
         assert result
 
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             with open(file_path, 'rt') as hook_file:
                 content = hook_file.read()
             assert content == non_rc_content
 
     def test_non_rc_hooks_are_replaced_with_force_flag(self):
         repo = self.create_dummy_repo('svn')
         hooks_path = os.path.join(repo.path, 'hooks')
         non_rc_content = 'echo "non rc hook"\n'
         for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
             self._create_fake_hook(
                 file_path, content=non_rc_content)
 
         result = hook_utils.install_svn_hooks(
             repo.path, force_create=True)
         assert result
         self.check_hooks(repo.path, )

vcsserver/tests/test_main_http.py

@@ -1,57 +1,57 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import mock
 import pytest
 
 from vcsserver import http_main
 from vcsserver.base import obfuscate_qs
 
 
 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
 def test_applies_largefiles_patch(patch_largefiles_capabilities):
     http_main.main({})
     patch_largefiles_capabilities.assert_called_once_with()
 
 
 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
 @mock.patch('vcsserver.http_main.MercurialFactory', None)
 @mock.patch(
     'vcsserver.hgpatches.patch_largefiles_capabilities',
     mock.Mock(side_effect=Exception("Must not be called")))
 def test_applies_largefiles_patch_only_if_mercurial_is_available():
     http_main.main({})
 
 
 @pytest.mark.parametrize('given, expected', [
     ('bad', 'bad'),
     ('query&foo=bar', 'query&foo=bar'),
     ('equery&auth_token=bar', 'equery&auth_token=*****'),
     ('a;b;c;query&foo=bar&auth_token=secret',
      'a&b&c&query&foo=bar&auth_token=*****'),
     ('', ''),
     (None, None),
     ('foo=bar', 'foo=bar'),
     ('auth_token=secret', 'auth_token=*****'),
     ('auth_token=secret&api_key=secret2',
      'auth_token=*****&api_key=*****'),
     ('auth_token=secret&api_key=secret2&param=value',
      'auth_token=*****&api_key=*****&param=value'),
 ])
 def test_obfuscate_qs(given, expected):
     assert expected == obfuscate_qs(given)

vcsserver/tests/test_pygrack.py

@@ -1,249 +1,249 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 
 import dulwich.protocol
 import mock
 import pytest
 import webob
 import webtest
 
 from vcsserver import hooks, pygrack
 
 # pylint: disable=redefined-outer-name,protected-access
 
 
 @pytest.fixture()
 def pygrack_instance(tmpdir):
     """
     Creates a pygrack app instance.
 
     Right now, it does not much helpful regarding the passed directory.
     It just contains the required folders to pass the signature test.
     """
     for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
         tmpdir.mkdir(dir_name)
 
     return pygrack.GitRepository('repo_name', str(tmpdir), 'git', False, {})
 
 
 @pytest.fixture()
 def pygrack_app(pygrack_instance):
     """
     Creates a pygrack app wrapped in webtest.TestApp.
     """
     return webtest.TestApp(pygrack_instance)
 
 
 def test_invalid_service_info_refs_returns_403(pygrack_app):
     response = pygrack_app.get('/info/refs?service=git-upload-packs',
                                expect_errors=True)
 
     assert response.status_int == 403
 
 
 def test_invalid_endpoint_returns_403(pygrack_app):
     response = pygrack_app.post('/git-upload-packs', expect_errors=True)
 
     assert response.status_int == 403
 
 
 @pytest.mark.parametrize('sideband', [
     'side-band-64k',
     'side-band',
     'side-band no-progress',
 ])
 def test_pre_pull_hook_fails_with_sideband(pygrack_app, sideband):
     request = ''.join([
         '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ',
         'multi_ack %s ofs-delta\n' % sideband,
         '0000',
         '0009done\n',
     ])
     with mock.patch('vcsserver.hooks.git_pre_pull',
                     return_value=hooks.HookResponse(1, 'foo')):
         response = pygrack_app.post(
             '/git-upload-pack', params=request,
             content_type='application/x-git-upload-pack')
 
     data = io.BytesIO(response.body)
     proto = dulwich.protocol.Protocol(data.read, None)
     packets = list(proto.read_pkt_seq())
 
     expected_packets = [
         'NAK\n', '\x02foo', '\x02Pre pull hook failed: aborting\n',
         '\x01' + pygrack.GitRepository.EMPTY_PACK,
     ]
     assert packets == expected_packets
 
 
 def test_pre_pull_hook_fails_no_sideband(pygrack_app):
     request = ''.join([
         '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
         'multi_ack ofs-delta\n'
         '0000',
         '0009done\n',
     ])
     with mock.patch('vcsserver.hooks.git_pre_pull',
                     return_value=hooks.HookResponse(1, 'foo')):
         response = pygrack_app.post(
             '/git-upload-pack', params=request,
             content_type='application/x-git-upload-pack')
 
     assert response.body == pygrack.GitRepository.EMPTY_PACK
 
 
 def test_pull_has_hook_messages(pygrack_app):
     request = ''.join([
         '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
         'multi_ack side-band-64k ofs-delta\n'
         '0000',
         '0009done\n',
     ])
     with mock.patch('vcsserver.hooks.git_pre_pull',
                     return_value=hooks.HookResponse(0, 'foo')):
         with mock.patch('vcsserver.hooks.git_post_pull',
                         return_value=hooks.HookResponse(1, 'bar')):
             with mock.patch('vcsserver.subprocessio.SubprocessIOChunker',
                             return_value=['0008NAK\n0009subp\n0000']):
                 response = pygrack_app.post(
                     '/git-upload-pack', params=request,
                     content_type='application/x-git-upload-pack')
 
     data = io.BytesIO(response.body)
     proto = dulwich.protocol.Protocol(data.read, None)
     packets = list(proto.read_pkt_seq())
 
     assert packets == ['NAK\n', '\x02foo', 'subp\n', '\x02bar']
 
 
 def test_get_want_capabilities(pygrack_instance):
     data = io.BytesIO(
         '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
         'multi_ack side-band-64k ofs-delta\n00000009done\n')
 
     request = webob.Request({
         'wsgi.input': data,
         'REQUEST_METHOD': 'POST',
         'webob.is_body_seekable': True
     })
 
     capabilities = pygrack_instance._get_want_capabilities(request)
 
     assert capabilities == frozenset(
         ('ofs-delta', 'multi_ack', 'side-band-64k'))
     assert data.tell() == 0
 
 
 @pytest.mark.parametrize('data,capabilities,expected', [
     ('foo', [], []),
     ('', ['side-band-64k'], []),
     ('', ['side-band'], []),
     ('foo', ['side-band-64k'], ['0008\x02foo']),
     ('foo', ['side-band'], ['0008\x02foo']),
     ('f'*1000, ['side-band-64k'], ['03ed\x02' + 'f' * 1000]),
     ('f'*1000, ['side-band'], ['03e8\x02' + 'f' * 995, '000a\x02fffff']),
     ('f'*65520, ['side-band-64k'], ['fff0\x02' + 'f' * 65515, '000a\x02fffff']),
     ('f'*65520, ['side-band'], ['03e8\x02' + 'f' * 995] * 65 + ['0352\x02' + 'f' * 845]),
 ], ids=[
     'foo-empty',
     'empty-64k', 'empty',
     'foo-64k', 'foo',
     'f-1000-64k', 'f-1000',
     'f-65520-64k', 'f-65520'])
 def test_get_messages(pygrack_instance, data, capabilities, expected):
     messages = pygrack_instance._get_messages(data, capabilities)
 
     assert messages == expected
 
 
 @pytest.mark.parametrize('response,capabilities,pre_pull_messages,post_pull_messages', [
     # Unexpected response
     ('unexpected_response', ['side-band-64k'], 'foo', 'bar'),
     # No sideband
     ('no-sideband', [], 'foo', 'bar'),
     # No messages
     ('no-messages', ['side-band-64k'], '', ''),
 ])
 def test_inject_messages_to_response_nothing_to_do(
         pygrack_instance, response, capabilities, pre_pull_messages,
         post_pull_messages):
     new_response = pygrack_instance._inject_messages_to_response(
         response, capabilities, pre_pull_messages, post_pull_messages)
 
     assert new_response == response
 
 
 @pytest.mark.parametrize('capabilities', [
     ['side-band'],
     ['side-band-64k'],
 ])
 def test_inject_messages_to_response_single_element(pygrack_instance,
                                                     capabilities):
     response = ['0008NAK\n0009subp\n0000']
     new_response = pygrack_instance._inject_messages_to_response(
         response, capabilities, 'foo', 'bar')
 
     expected_response = [
         '0008NAK\n', '0008\x02foo', '0009subp\n', '0008\x02bar', '0000']
 
     assert new_response == expected_response
 
 
 @pytest.mark.parametrize('capabilities', [
     ['side-band'],
     ['side-band-64k'],
 ])
 def test_inject_messages_to_response_multi_element(pygrack_instance,
                                                    capabilities):
     response = [
         '0008NAK\n000asubp1\n', '000asubp2\n', '000asubp3\n', '000asubp4\n0000']
     new_response = pygrack_instance._inject_messages_to_response(
         response, capabilities, 'foo', 'bar')
 
     expected_response = [
         '0008NAK\n', '0008\x02foo', '000asubp1\n', '000asubp2\n', '000asubp3\n',
         '000asubp4\n', '0008\x02bar', '0000'
     ]
 
     assert new_response == expected_response
 
 
 def test_build_failed_pre_pull_response_no_sideband(pygrack_instance):
     response = pygrack_instance._build_failed_pre_pull_response([], 'foo')
 
     assert response == [pygrack.GitRepository.EMPTY_PACK]
 
 
 @pytest.mark.parametrize('capabilities', [
     ['side-band'],
     ['side-band-64k'],
     ['side-band-64k', 'no-progress'],
 ])
 def test_build_failed_pre_pull_response(pygrack_instance, capabilities):
     response = pygrack_instance._build_failed_pre_pull_response(
         capabilities, 'foo')
 
     expected_response = [
         '0008NAK\n', '0008\x02foo', '0024\x02Pre pull hook failed: aborting\n',
         '%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5,
                         pygrack.GitRepository.EMPTY_PACK),
         '0000',
     ]
 
     assert response == expected_response

vcsserver/tests/test_scm_app.py

@@ -1,86 +1,86 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 
 import mercurial.hg
 import mercurial.ui
 import mercurial.error
 import mock
 import pytest
 import webtest
 
 from vcsserver import scm_app
 
 
 def test_hg_does_not_accept_invalid_cmd(tmpdir):
     repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
     app = webtest.TestApp(scm_app.HgWeb(repo))
 
     response = app.get('/repo?cmd=invalidcmd', expect_errors=True)
 
     assert response.status_int == 400
 
 
 def test_create_hg_wsgi_app_requirement_error(tmpdir):
     repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
     config = (
         ('paths', 'default', ''),
     )
     with mock.patch('vcsserver.scm_app.HgWeb') as hgweb_mock:
         hgweb_mock.side_effect = mercurial.error.RequirementError()
         with pytest.raises(Exception):
             scm_app.create_hg_wsgi_app(str(tmpdir), repo, config)
 
 
 def test_git_returns_not_found(tmpdir):
     app = webtest.TestApp(
         scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
 
     response = app.get('/repo_name/inforefs?service=git-upload-pack',
                        expect_errors=True)
 
     assert response.status_int == 404
 
 
 def test_git(tmpdir):
     for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
         tmpdir.mkdir(dir_name)
 
     app = webtest.TestApp(
         scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
 
     # We set service to git-upload-packs to trigger a 403
     response = app.get('/repo_name/inforefs?service=git-upload-packs',
                        expect_errors=True)
 
     assert response.status_int == 403
 
 
 def test_git_fallbacks_to_git_folder(tmpdir):
     tmpdir.mkdir('.git')
     for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
         tmpdir.mkdir(os.path.join('.git', dir_name))
 
     app = webtest.TestApp(
         scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
 
     # We set service to git-upload-packs to trigger a 403
     response = app.get('/repo_name/inforefs?service=git-upload-packs',
                        expect_errors=True)
 
     assert response.status_int == 403

vcsserver/tests/test_server.py

@@ -1,39 +1,39 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 
 import mock
 import pytest
 
 from vcsserver.server import VcsServer
 
 
 def test_provides_the_pid(server):
     pid = server.get_pid()
     assert pid == os.getpid()
 
 
 def test_allows_to_trigger_the_garbage_collector(server):
     with mock.patch('gc.collect') as collect:
         server.run_gc()
     assert collect.called
 
 
 @pytest.fixture
 def server():
     return VcsServer()

vcsserver/tests/test_subprocessio.py

@@ -1,155 +1,155 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 
 import pytest
 
 from vcsserver import subprocessio
 
 
 class KindaFilelike(object):  # pragma: no cover
 
     def __init__(self, data, size):
         chunks = size / len(data)
 
         self.stream = self._get_stream(data, chunks)
 
     def _get_stream(self, data, chunks):
         for x in xrange(chunks):
             yield data
 
     def read(self, n):
 
         buffer_stream = ''
         for chunk in self.stream:
             buffer_stream += chunk
             if len(buffer_stream) >= n:
                 break
 
         # self.stream = self.bytes[n:]
         return buffer_stream
 
 
 @pytest.fixture(scope='module')
 def environ():
     """Delete coverage variables, as they make the tests fail."""
     env = dict(os.environ)
     for key in env.keys():
         if key.startswith('COV_CORE_'):
             del env[key]
 
     return env
 
 
 def _get_python_args(script):
     return [sys.executable, '-c', 'import sys; import time; import shutil; ' + script]
 
 
 def test_raise_exception_on_non_zero_return_code(environ):
     args = _get_python_args('sys.exit(1)')
     with pytest.raises(EnvironmentError):
         list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
 
 def test_does_not_fail_on_non_zero_return_code(environ):
     args = _get_python_args('sys.exit(1)')
     output = ''.join(
         subprocessio.SubprocessIOChunker(
             args, shell=False, fail_on_return_code=False, env=environ
         )
     )
 
     assert output == ''
 
 
 def test_raise_exception_on_stderr(environ):
     args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
     with pytest.raises(EnvironmentError) as excinfo:
         list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
     assert 'exited due to an error:\nX' in str(excinfo.value)
 
 
 def test_does_not_fail_on_stderr(environ):
     args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
     output = ''.join(
         subprocessio.SubprocessIOChunker(
             args, shell=False, fail_on_stderr=False, env=environ
         )
     )
 
     assert output == ''
 
 
 @pytest.mark.parametrize('size', [1, 10 ** 5])
 def test_output_with_no_input(size, environ):
     print(type(environ))
     data = 'X'
     args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size))
     output = ''.join(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
 
     assert output == data * size
 
 
 @pytest.mark.parametrize('size', [1, 10 ** 5])
 def test_output_with_no_input_does_not_fail(size, environ):
     data = 'X'
     args = _get_python_args('sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
     output = ''.join(
         subprocessio.SubprocessIOChunker(
             args, shell=False, fail_on_return_code=False, env=environ
         )
     )
 
     print("{} {}".format(len(data * size), len(output)))
     assert output == data * size
 
 
 @pytest.mark.parametrize('size', [1, 10 ** 5])
 def test_output_with_input(size, environ):
     data_len = size
     inputstream = KindaFilelike('X', size)
 
     # This acts like the cat command.
     args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
     output = ''.join(
         subprocessio.SubprocessIOChunker(
             args, shell=False, inputstream=inputstream, env=environ
         )
     )
 
     assert len(output) == data_len
 
 
 @pytest.mark.parametrize('size', [1, 10 ** 5])
 def test_output_with_input_skipping_iterator(size, environ):
     data_len = size
     inputstream = KindaFilelike('X', size)
 
     # This acts like the cat command.
     args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
 
     # Note: assigning the chunker makes sure that it is not deleted too early
     chunker = subprocessio.SubprocessIOChunker(
         args, shell=False, inputstream=inputstream, env=environ
     )
     output = ''.join(chunker.output)
 
     assert len(output) == data_len

vcsserver/tests/test_svn.py

@@ -1,87 +1,87 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import mock
 import pytest
 import sys
 
 
 class MockPopen(object):
     def __init__(self, stderr):
         self.stdout = io.BytesIO('')
         self.stderr = io.BytesIO(stderr)
         self.returncode = 1
 
     def wait(self):
         pass
 
 
 INVALID_CERTIFICATE_STDERR = '\n'.join([
     'svnrdump: E230001: Unable to connect to a repository at URL url',
     'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
 ])
 
 
 @pytest.mark.parametrize('stderr,expected_reason', [
     (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
     ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
 @pytest.mark.xfail(sys.platform == "cygwin",
                    reason="SVN not packaged for Cygwin")
 def test_import_remote_repository_certificate_error(stderr, expected_reason):
     from vcsserver import svn
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
     remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     with mock.patch('subprocess.Popen',
                     return_value=MockPopen(stderr)):
         with pytest.raises(Exception) as excinfo:
             remote.import_remote_repository({'path': 'path'}, 'url')
 
     expected_error_args = (
         'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason),)
 
     assert excinfo.value.args == expected_error_args
 
 
 def test_svn_libraries_can_be_imported():
     import svn
     import svn.client
     assert svn.client is not None
 
 
 @pytest.mark.parametrize('example_url, parts', [
     ('http://server.com', (None, None, 'http://server.com')),
     ('http://[e-mail]', ('user', None, 'http://[e-mail]')),
     ('http://user:[e-mail]', ('user', 'pass', 'http://user:[e-mail]')),
     ('<script>', (None, None, '<script>')),
     ('http://', (None, None, 'http://')),
 ])
 def test_username_password_extraction_from_url(example_url, parts):
     from vcsserver import svn
 
     factory = mock.Mock()
     factory.repo = mock.Mock(return_value=mock.Mock())
 
     remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     assert remote.get_url_and_credentials(example_url) == parts

vcsserver/tests/test_wsgi_app_caller.py

@@ -1,96 +1,96 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import wsgiref.simple_server
 import wsgiref.validate
 
 from vcsserver import wsgi_app_caller
 
 
 # pylint: disable=protected-access,too-many-public-methods
 
 
 @wsgiref.validate.validator
 def demo_app(environ, start_response):
     """WSGI app used for testing."""
     data = [
         'Hello World!\n',
         'input_data=%s\n' % environ['wsgi.input'].read(),
     ]
     for key, value in sorted(environ.items()):
         data.append('%s=%s\n' % (key, value))
 
     write = start_response("200 OK", [('Content-Type', 'text/plain')])
     write('Old school write method\n')
     write('***********************\n')
     return data
 
 
 BASE_ENVIRON = {
     'REQUEST_METHOD': 'GET',
     'SERVER_NAME': 'localhost',
     'SERVER_PORT': '80',
     'SCRIPT_NAME': '',
     'PATH_INFO': '/',
     'QUERY_STRING': '',
     'foo.var': 'bla',
 }
 
 
 def test_complete_environ():
     environ = dict(BASE_ENVIRON)
     data = "data"
     wsgi_app_caller._complete_environ(environ, data)
     wsgiref.validate.check_environ(environ)
 
     assert data == environ['wsgi.input'].read()
 
 
 def test_start_response():
     start_response = wsgi_app_caller._StartResponse()
     status = '200 OK'
     headers = [('Content-Type', 'text/plain')]
     start_response(status, headers)
 
     assert status == start_response.status
     assert headers == start_response.headers
 
 
 def test_start_response_with_error():
     start_response = wsgi_app_caller._StartResponse()
     status = '500 Internal Server Error'
     headers = [('Content-Type', 'text/plain')]
     start_response(status, headers, (None, None, None))
 
     assert status == start_response.status
     assert headers == start_response.headers
 
 
 def test_wsgi_app_caller():
     caller = wsgi_app_caller.WSGIAppCaller(demo_app)
     environ = dict(BASE_ENVIRON)
     input_data = 'some text'
     responses, status, headers = caller.handle(environ, input_data)
     response = ''.join(responses)
 
     assert status == '200 OK'
     assert headers == [('Content-Type', 'text/plain')]
     assert response.startswith(
         'Old school write method\n***********************\n')
     assert 'Hello World!\n' in response
     assert 'foo.var=bla\n' in response
     assert 'input_data=%s\n' % input_data in response

vcsserver/tweens/__init__.py

@@ -1,19 +1,19 b''
 # -*- coding: utf-8 -*-
 
-# Copyright (C) 2016-2019 RhodeCode GmbH
+# Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/

vcsserver/tweens/request_wrapper.py

@@ -1,64 +1,64 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import time
 import logging
 
 import vcsserver
 from vcsserver.utils import safe_str
 
 
 log = logging.getLogger(__name__)
 
 
 def get_access_path(request):
     environ = request.environ
     return environ.get('PATH_INFO')
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 class RequestWrapperTween(object):
     def __init__(self, handler, registry):
         self.handler = handler
         self.registry = registry
 
         # one-time configuration code goes here
 
     def __call__(self, request):
         start = time.time()
         try:
             response = self.handler(request)
         finally:
             end = time.time()
             total = end - start
             count = request.request_count()
             _ver_ = vcsserver.__version__
             log.info(
                 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
                 count, '127.0.0.1', request.environ.get('REQUEST_METHOD'),
                 safe_str(get_access_path(request)), total, get_user_agent(request.environ), _ver_)
 
         return response
 
 
 def includeme(config):
     config.add_tween(
         'vcsserver.tweens.request_wrapper.RequestWrapperTween',
     )

vcsserver/utils.py

@@ -1,110 +1,110 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 import logging
 import hashlib
 
 log = logging.getLogger(__name__)
 
 
 def safe_int(val, default=None):
     """
     Returns int() of val if val is not convertable to int use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def safe_str(unicode_, to_encoding=None):
     """
     safe str function. Does few trick to turn unicode_ into string
 
     :param unicode_: unicode to encode
     :param to_encoding: encode to this type UTF8 default
     :rtype: str
     :returns: str object
     """
     to_encoding = to_encoding or ['utf8']
     # if it's not basestr cast to str
     if not isinstance(unicode_, basestring):
         return str(unicode_)
 
     if isinstance(unicode_, str):
         return unicode_
 
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return unicode_.encode(enc)
         except UnicodeEncodeError:
             pass
 
     return unicode_.encode(to_encoding[0], 'replace')
 
 
 def safe_unicode(str_, from_encoding=None):
     """
     safe unicode function. Does few trick to turn str_ into unicode
 
     :param str_: string to decode
     :param from_encoding: encode from this type UTF8 default
     :rtype: unicode
     :returns: unicode object
     """
     from_encoding = from_encoding or ['utf8']
 
     if isinstance(str_, unicode):
         return str_
 
     if not isinstance(from_encoding, (list, tuple)):
         from_encoding = [from_encoding]
 
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
 
     for enc in from_encoding:
         try:
             return unicode(str_, enc)
         except UnicodeDecodeError:
             pass
 
     return unicode(str_, from_encoding[0], 'replace')
 
 
 class AttributeDict(dict):
     def __getattr__(self, attr):
         return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
 def sha1(val):
     return hashlib.sha1(val).hexdigest()
 
 

vcsserver/vcs_base.py

@@ -1,32 +1,32 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
 class RemoteBase(object):
     EMPTY_COMMIT = '0' * 40
 
     @property
     def region(self):
         return self._factory._cache_region
 
     def _cache_on(self, wire):
         context = wire.get('context', '')
         context_uid = '{}'.format(context)
         repo_id = wire.get('repo_id', '')
         cache = wire.get('cache', True)
         cache_on = context and cache
         return cache_on, context_uid, repo_id

vcsserver/wsgi_app_caller.py

@@ -1,116 +1,116 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
-# Copyright (C) 2014-2019 RhodeCode GmbH
+# Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 """Extract the responses of a WSGI app."""
 
 __all__ = ('WSGIAppCaller',)
 
 import io
 import logging
 import os
 
 
 log = logging.getLogger(__name__)
 
 DEV_NULL = open(os.devnull)
 
 
 def _complete_environ(environ, input_data):
     """Update the missing wsgi.* variables of a WSGI environment.
 
     :param environ: WSGI environment to update
     :type environ: dict
     :param input_data: data to be read by the app
     :type input_data: str
     """
     environ.update({
         'wsgi.version': (1, 0),
         'wsgi.url_scheme': 'http',
         'wsgi.multithread': True,
         'wsgi.multiprocess': True,
         'wsgi.run_once': False,
         'wsgi.input': io.BytesIO(input_data),
         'wsgi.errors': DEV_NULL,
     })
 
 
 # pylint: disable=too-few-public-methods
 class _StartResponse(object):
     """Save the arguments of a start_response call."""
 
     __slots__ = ['status', 'headers', 'content']
 
     def __init__(self):
         self.status = None
         self.headers = None
         self.content = []
 
     def __call__(self, status, headers, exc_info=None):
         # TODO(skreft): do something meaningful with the exc_info
         exc_info = None  # avoid dangling circular reference
         self.status = status
         self.headers = headers
 
         return self.write
 
     def write(self, content):
         """Write method returning when calling this object.
 
         All the data written is then available in content.
         """
         self.content.append(content)
 
 
 class WSGIAppCaller(object):
     """Calls a WSGI app."""
 
     def __init__(self, app):
         """
         :param app: WSGI app to call
         """
         self.app = app
 
     def handle(self, environ, input_data):
         """Process a request with the WSGI app.
 
         The returned data of the app is fully consumed into a list.
 
         :param environ: WSGI environment to update
         :type environ: dict
         :param input_data: data to be read by the app
         :type input_data: str
 
         :returns: a tuple with the contents, status and headers
         :rtype: (list<str>, str, list<(str, str)>)
         """
         _complete_environ(environ, input_data)
         start_response = _StartResponse()
         log.debug("Calling wrapped WSGI application")
         responses = self.app(environ, start_response)
         responses_list = list(responses)
         existing_responses = start_response.content
         if existing_responses:
             log.debug(
                 "Adding returned response to response written via write()")
             existing_responses.extend(responses_list)
             responses_list = existing_responses
         if hasattr(responses, 'close'):
             log.debug("Closing iterator from WSGI application")
             responses.close()
 
         log.debug("Handling of WSGI request done, returning response")
         return responses_list, start_response.status, start_response.headers