release: Merge default into stable for release preparation

pkgs/subversion-1.9.4-darwin.patch

@@ -0,0 +1,63 b''
+diff -rup subversion-1.9.4-orig/subversion/include/svn_auth.h subversion-1.9.4/subversion/include/svn_auth.h
+--- subversion-1.9.4-orig/subversion/include/svn_auth.h	2015-02-13 12:17:40.000000000 +0100
++++ subversion-1.9.4/subversion/include/svn_auth.h	2016-09-21 12:55:27.000000000 +0200
+@@ -943,7 +943,7 @@ svn_auth_get_windows_ssl_server_trust_pr
+ 
+ #endif /* WIN32 && !__MINGW32__ || DOXYGEN */
+ 
+-#if defined(DARWIN) || defined(DOXYGEN)
++#if defined(SVN_HAVE_KEYCHAIN_SERVICES) || defined(DOXYGEN)
+ /**
+  * Set @a *provider to an authentication provider of type @c
+  * svn_auth_cred_simple_t that gets/sets information from the user's
+@@ -984,7 +984,7 @@ void
+ svn_auth_get_keychain_ssl_client_cert_pw_provider(
+   svn_auth_provider_object_t **provider,
+   apr_pool_t *pool);
+-#endif /* DARWIN || DOXYGEN */
++#endif /* SVN_HAVE_KEYCHAIN_SERVICES || DOXYGEN */
+ 
+ /* Note that the gnome keyring unlock prompt related items below must be
+  * declared for all platforms in order to allow SWIG interfaces to be
+diff -rup subversion-1.9.4-orig/subversion/libsvn_subr/auth.h subversion-1.9.4/subversion/libsvn_subr/auth.h
+--- subversion-1.9.4-orig/subversion/libsvn_subr/auth.h	2015-08-27 06:00:31.000000000 +0200
++++ subversion-1.9.4/subversion/libsvn_subr/auth.h	2016-09-21 12:56:20.000000000 +0200
+@@ -103,7 +103,7 @@ svn_auth__get_windows_ssl_server_trust_p
+   apr_pool_t *pool);
+ #endif /* WIN32 && !__MINGW32__ || DOXYGEN */
+ 
+-#if defined(DARWIN) || defined(DOXYGEN)
++#if defined(SVN_HAVE_KEYCHAIN_SERVICES) || defined(DOXYGEN)
+ /**
+  * Set @a *provider to an authentication provider of type @c
+  * svn_auth_cred_simple_t that gets/sets information from the user's
+@@ -134,7 +134,7 @@ void
+ svn_auth__get_keychain_ssl_client_cert_pw_provider(
+   svn_auth_provider_object_t **provider,
+   apr_pool_t *pool);
+-#endif /* DARWIN || DOXYGEN */
++#endif /* SVN_HAVE_KEYCHAIN_SERVICES || DOXYGEN */
+ 
+ #if !defined(WIN32) || defined(DOXYGEN)
+ /**
+diff -rup subversion-1.9.4-orig/subversion/libsvn_subr/deprecated.c subversion-1.9.4/subversion/libsvn_subr/deprecated.c
+--- subversion-1.9.4-orig/subversion/libsvn_subr/deprecated.c	2015-08-27 06:00:31.000000000 +0200
++++ subversion-1.9.4/subversion/libsvn_subr/deprecated.c	2016-09-21 12:57:08.000000000 +0200
+@@ -1479,7 +1479,7 @@ svn_auth_get_windows_ssl_server_trust_pr
+ #endif /* WIN32 && !__MINGW32__ */
+ 
+ /*** From macos_keychain.c ***/
+-#if defined(DARWIN)
++#if defined(SVN_HAVE_KEYCHAIN_SERVICES)
+ void
+ svn_auth_get_keychain_simple_provider(svn_auth_provider_object_t **provider,
+                                       apr_pool_t *pool)
+@@ -1494,7 +1494,7 @@ svn_auth_get_keychain_ssl_client_cert_pw
+ {
+   svn_auth__get_keychain_ssl_client_cert_pw_provider(provider, pool);
+ }
+-#endif /* DARWIN */
++#endif /* SVN_HAVE_KEYCHAIN_SERVICES */
+ 
+ #if !defined(WIN32)
+ void

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 4.4.2
+current_version = 4.5.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]

.release.cfg

@@ -5,12 +5,10 b' done = false'
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.4.2
+state = in_progress
+version = 4.5.0
 

configs/production_http.ini

@@ -3,22 +3,6 b''
 #                                                                              #
 ################################################################################
 
-[app:main]
-use = egg:rhodecode-vcsserver
-
-pyramid.default_locale_name = en
-pyramid.includes =
-
-# default locale used by VCS systems
-locale = en_US.UTF-8
-
-# cache regions, please don't change
-beaker.cache.regions = repo_object
-beaker.cache.repo_object.type = memorylru
-beaker.cache.repo_object.max_items = 100
-# cache auto-expires after N seconds
-beaker.cache.repo_object.expire = 300
-beaker.cache.repo_object.enabled = true
 
 [server:main]
 ## COMMON ##
@@ -29,7 +13,7 b' port = 9900'
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
-## run with gunicorn --log-config <inifile.ini> --paste <inifile.ini>
+## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
 use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
@@ -52,6 +36,22 b' max_requests_jitter = 30'
 timeout = 21600
 
 
+[app:main]
+use = egg:rhodecode-vcsserver
+
+pyramid.default_locale_name = en
+pyramid.includes =
+
+## default locale used by VCS systems
+locale = en_US.UTF-8
+
+# cache regions, please don't change
+beaker.cache.regions = repo_object
+beaker.cache.repo_object.type = memorylru
+beaker.cache.repo_object.max_items = 100
+# cache auto-expires after N seconds
+beaker.cache.repo_object.expire = 300
+beaker.cache.repo_object.enabled = true
 
 
 ################################

default.nix

@@ -16,12 +16,19 b' let'
   pkgs = pkgs_.overridePackages (self: super: {
     # Override subversion derivation to
     #  - activate python bindings
-    #  - set version to 1.8
-    subversion = super.subversion18.override {
-       httpSupport = true;
-       pythonBindings = true;
-       python = self.python27Packages.python;
-    };
+    subversion = let
+      subversionWithPython = super.subversion.override {
+        httpSupport = true;
+        pythonBindings = true;
+        python = self.python27Packages.python;
+      };
+    in pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: {
+      patches = (oldAttrs.patches or []) ++
+        pkgs.lib.optionals pkgs.stdenv.isDarwin [
+          # johbo: "import svn.client" fails on darwin currently.
+          ./pkgs/subversion-1.9.4-darwin.patch
+        ];
+    });
   });
 
   inherit (pkgs.lib) fix extends;
@@ -86,21 +93,6 b' let'
         pythonPackages = self;
       };
 
-      # Somewhat snappier setup of the development environment
-      # TODO: move into shell.nix
-      # TODO: think of supporting a stable path again, so that multiple shells
-      #       can share it.
-      shellHook = ''
-        # Set locale
-        export LC_ALL="en_US.UTF-8"
-
-        tmp_path=$(mktemp -d)
-        export PATH="$tmp_path/bin:$PATH"
-        export PYTHONPATH="$tmp_path/${self.python.sitePackages}:$PYTHONPATH"
-        mkdir -p $tmp_path/${self.python.sitePackages}
-        python setup.py develop --prefix $tmp_path --allow-hosts ""
-      '';
-
       # Add VCSServer bin directory to path so that tests can find 'vcsserver'.
       preCheck = ''
         export PATH="$out/bin:$PATH"

pkgs/python-packages-overrides.nix

@@ -13,7 +13,8 b' in'
 self: super: {
 
   subvertpy = super.subvertpy.override (attrs: {
-    SVN_PREFIX = "${pkgs.subversion}";
+    # TODO: johbo: Remove the "or" once we drop 16.03 support
+    SVN_PREFIX = "${pkgs.subversion.dev or pkgs.subversion}";
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.aprutil
       pkgs.subversion

pkgs/python-packages.nix

@@ -1,3 +1,6 b''
+# Generated by pip2nix 0.4.0.dev1
+# See https://github.com/johbo/pip2nix
+
 {
   Beaker = super.buildPythonPackage {
     name = "Beaker-1.7.0";
@@ -26,13 +29,13 b''
     };
   };
   Mako = super.buildPythonPackage {
-    name = "Mako-1.0.4";
+    name = "Mako-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/7a/ae/925434246ee90b42e8ef57d3b30a0ab7caf9a2de3e449b876c56dcb48155/Mako-1.0.4.tar.gz";
-      md5 = "c5fc31a323dd4990683d2f2da02d4e20";
+      url = "https://pypi.python.org/packages/56/4b/cb75836863a6382199aefb3d3809937e21fa4cb0db15a4f4ba0ecc2e7e8e/Mako-1.0.6.tar.gz";
+      md5 = "a28e22a339080316b2acc352b9ee631c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -103,6 +106,19 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  backports.shutil-get-terminal-size = super.buildPythonPackage {
+    name = "backports.shutil-get-terminal-size-1.0.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
+      md5 = "03267762480bd86b50580dc19dff3c66";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   configobj = super.buildPythonPackage {
     name = "configobj-5.0.6";
     buildInputs = with self; [];
@@ -116,6 +132,19 b''
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  decorator = super.buildPythonPackage {
+    name = "decorator-4.0.10";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz";
+      md5 = "434b57fdc3230c500716c5aff8896100";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
+    };
+  };
   dulwich = super.buildPythonPackage {
     name = "dulwich-0.13.0";
     buildInputs = with self; [];
@@ -129,6 +158,19 b''
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
+  enum34 = super.buildPythonPackage {
+    name = "enum34-1.1.6";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
+      md5 = "5f13a0841a61f7fc295c514490d120d0";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   greenlet = super.buildPythonPackage {
     name = "greenlet-0.4.7";
     buildInputs = with self; [];
@@ -181,6 +223,45 b''
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
+  ipdb = super.buildPythonPackage {
+    name = "ipdb-0.10.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [ipython setuptools];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/eb/0a/0a37dc19572580336ad3813792c0d18c8d7117c2d66fc63c501f13a7a8f8/ipdb-0.10.1.tar.gz";
+      md5 = "4aeab65f633ddc98ebdb5eebf08dc713";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  ipython = super.buildPythonPackage {
+    name = "ipython-5.1.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [setuptools decorator pickleshare simplegeneric traitlets prompt-toolkit pygments pexpect backports.shutil-get-terminal-size pathlib2 pexpect];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
+      md5 = "47c8122420f65b58784cb4b9b4af35e3";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  ipython-genutils = super.buildPythonPackage {
+    name = "ipython-genutils-0.1.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/71/b7/a64c71578521606edbbce15151358598f3dfb72a3431763edc2baf19e71f/ipython_genutils-0.1.0.tar.gz";
+      md5 = "9a8afbe0978adbcbfcb3b35b2d015a56";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   mercurial = super.buildPythonPackage {
     name = "mercurial-3.8.4";
     buildInputs = with self; [];
@@ -220,6 +301,71 b''
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  pathlib2 = super.buildPythonPackage {
+    name = "pathlib2-2.1.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [six];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/c9/27/8448b10d8440c08efeff0794adf7d0ed27adb98372c70c7b38f3947d4749/pathlib2-2.1.0.tar.gz";
+      md5 = "38e4f58b4d69dfcb9edb49a54a8b28d2";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
+  pexpect = super.buildPythonPackage {
+    name = "pexpect-4.2.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [ptyprocess];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/e8/13/d0b0599099d6cd23663043a2a0bb7c61e58c6ba359b2656e6fb000ef5b98/pexpect-4.2.1.tar.gz";
+      md5 = "3694410001a99dff83f0b500a1ca1c95";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
+    };
+  };
+  pickleshare = super.buildPythonPackage {
+    name = "pickleshare-0.7.4";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [pathlib2];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz";
+      md5 = "6a9e5dd8dfc023031f6b7b3f824cab12";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
+  prompt-toolkit = super.buildPythonPackage {
+    name = "prompt-toolkit-1.0.9";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [six wcwidth];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/83/14/5ac258da6c530eca02852ee25c7a9ff3ca78287bb4c198d0d0055845d856/prompt_toolkit-1.0.9.tar.gz";
+      md5 = "a39f91a54308fb7446b1a421c11f227c";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  ptyprocess = super.buildPythonPackage {
+    name = "ptyprocess-0.5.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/db/d7/b465161910f3d1cef593c5e002bff67e0384898f597f1a7fdc8db4c02bf6/ptyprocess-0.5.1.tar.gz";
+      md5 = "94e537122914cc9ec9c1eadcd36e73a1";
+    };
+    meta = {
+      license = [  ];
+    };
+  };
   py = super.buildPythonPackage {
     name = "py-1.4.29";
     buildInputs = with self; [];
@@ -233,6 +379,19 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  pygments = super.buildPythonPackage {
+    name = "pygments-2.1.3";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/b8/67/ab177979be1c81bc99c8d0592ef22d547e70bb4c6815c383286ed5dec504/Pygments-2.1.3.tar.gz";
+      md5 = "ed3fba2467c8afcda4d317e4ef2c6150";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   pyramid = super.buildPythonPackage {
     name = "pyramid-1.6.1";
     buildInputs = with self; [];
@@ -299,8 +458,8 b''
     };
   };
   rhodecode-vcsserver = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.4.2";
-    buildInputs = with self; [mock pytest WebTest];
+    name = "rhodecode-vcsserver-4.5.0";
+    buildInputs = with self; [mock pytest pytest-sugar WebTest];
     doCheck = true;
     propagatedBuildInputs = with self; [configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid Pyro4 simplejson subprocess32 waitress WebOb];
     src = ./.;
@@ -334,6 +493,19 b''
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  simplegeneric = super.buildPythonPackage {
+    name = "simplegeneric-0.8.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
+      md5 = "f9c1fab00fd981be588fc32759f474e3";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.zpt21 ];
+    };
+  };
   simplejson = super.buildPythonPackage {
     name = "simplejson-3.7.2";
     buildInputs = with self; [];
@@ -344,7 +516,7 b''
       md5 = "a5fc7d05d4cb38492285553def5d4b46";
     };
     meta = {
-      license = [ pkgs.lib.licenses.mit pkgs.lib.licenses.afl21 ];
+      license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   six = super.buildPythonPackage {
@@ -386,6 +558,19 b''
       license = [ pkgs.lib.licenses.lgpl21Plus ];
     };
   };
+  traitlets = super.buildPythonPackage {
+    name = "traitlets-4.3.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [ipython-genutils six decorator enum34];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/b1/d6/5b5aa6d5c474691909b91493da1e8972e309c9f01ecfe4aeafd272eb3234/traitlets-4.3.1.tar.gz";
+      md5 = "dd0b1b6e5d31ce446d55a4b5e5083c98";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
   translationstring = super.buildPythonPackage {
     name = "translationstring-1.3";
     buildInputs = with self; [];
@@ -425,6 +610,19 b''
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
+  wcwidth = super.buildPythonPackage {
+    name = "wcwidth-0.1.7";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
+      md5 = "b3b6a0a08f0c8a34d1de8cf44150a4ad";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   wheel = super.buildPythonPackage {
     name = "wheel-0.29.0";
     buildInputs = with self; [];
@@ -467,5 +665,30 b''
 
 ### Test requirements
 
-  
+  pytest-sugar = super.buildPythonPackage {
+    name = "pytest-sugar-0.7.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [pytest termcolor];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/03/97/05d988b4fa870e7373e8ee4582408543b9ca2bd35c3c67b569369c6f9c49/pytest-sugar-0.7.1.tar.gz";
+      md5 = "7400f7c11f3d572b2c2a3b60352d35fe";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  termcolor = super.buildPythonPackage {
+    name = "termcolor-1.1.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
+      md5 = "043e89644f8909d462fbbfa511c768df";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
 }

requirements.txt

@@ -3,6 +3,7 b' configobj==5.0.6'
 dulwich==0.13.0
 hgsubversion==1.8.6
 infrae.cache==1.0.1
+ipdb==0.10.1
 mercurial==3.8.4
 msgpack-python==0.4.6
 py==1.4.29

setup.py

@@ -74,6 +74,7 b' setup('
     tests_require=[
         'mock',
         'pytest',
+        'pytest-sugar',
         'WebTest',
     ],
     install_requires=[

shell.nix

@@ -1,18 +1,41 b''
 { pkgs ? import <nixpkgs> {}
-,  doCheck ? false
+, doCheck ? false
 }:
 
 let
+
   vcsserver = import ./default.nix {
-    inherit
-      doCheck
-      pkgs;
+    inherit pkgs doCheck;
   };
 
+  vcs-pythonPackages = vcsserver.pythonPackages;
+
 in vcsserver.override (attrs: {
 
   # Avoid that we dump any sources into the store when entering the shell and
   # make development a little bit more convenient.
   src = null;
 
+  buildInputs =
+    attrs.buildInputs ++
+    (with vcs-pythonPackages; [
+      ipdb
+    ]);
+
+  # Somewhat snappier setup of the development environment
+  # TODO: think of supporting a stable path again, so that multiple shells
+  #       can share it.
+  postShellHook = ''
+    # Set locale
+    export LC_ALL="en_US.UTF-8"
+
+    # Custom prompt to distinguish from other dev envs.
+    export PS1="\n\[\033[1;32m\][VCS-shell:\w]$\[\033[0m\] "
+
+    tmp_path=$(mktemp -d)
+    export PATH="$tmp_path/bin:$PATH"
+    export PYTHONPATH="$tmp_path/${vcs-pythonPackages.python.sitePackages}:$PYTHONPATH"
+    mkdir -p $tmp_path/${vcs-pythonPackages.python.sitePackages}
+    python setup.py develop --prefix $tmp_path --allow-hosts ""
+  '';
 })

tests/test_main.py

@@ -16,8 +16,10 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import mock
+import pytest
 
 from vcsserver import main
+from vcsserver.base import obfuscate_qs
 
 
 @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock())
@@ -34,3 +36,22 b' def test_applies_largefiles_patch(patch_'
     mock.Mock(side_effect=Exception("Must not be called")))
 def test_applies_largefiles_patch_only_if_mercurial_is_available():
     main.main([])
+
+
[e-mail]('given, expected', [
+    ('bad', 'bad'),
+    ('query&foo=bar', 'query&foo=bar'),
+    ('equery&auth_token=bar', 'equery&auth_token=*****'),
+    ('a;b;c;query&foo=bar&auth_token=secret',
+     'a&b&c&query&foo=bar&auth_token=*****'),
+    ('', ''),
+    (None, None),
+    ('foo=bar', 'foo=bar'),
+    ('auth_token=secret', 'auth_token=*****'),
+    ('auth_token=secret&api_key=secret2',
+     'auth_token=*****&api_key=*****'),
+    ('auth_token=secret&api_key=secret2&param=value',
+     'auth_token=*****&api_key=*****&param=value'),
+])
+def test_obfuscate_qs(given, expected):
+    assert expected == obfuscate_qs(given)

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.4.2
  No newline at end of file
+4.5.0
  No newline at end of file

vcsserver/base.py

@@ -16,7 +16,7 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
-
+import urlparse
 
 log = logging.getLogger(__name__)
 
@@ -69,3 +69,17 b' class RepoFactory(object):'
                 'INIT %s@%s repo object based on wire %s. Context: %s',
                 self.__class__.__name__, wire['path'], wire, context)
             return createfunc()
+
+
+def obfuscate_qs(query_string):
+    if query_string is None:
+        return None
+
+    parsed = []
+    for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
+        if k in ['auth_token', 'api_key']:
+            v = "*****"
+        parsed.append((k, v))
+
+    return '&'.join('{}{}'.format(
+        k, '={}'.format(v) if v else '') for k, v in parsed)

vcsserver/exceptions.py

@@ -25,6 +25,7 b' different error conditions.'
 """
 
 import functools
+from pyramid.httpexceptions import HTTPLocked
 
 
 def _make_exception(kind, *args):
@@ -54,3 +55,16 b' RequirementException = functools.partial'
 UnhandledException = functools.partial(_make_exception, 'unhandled')
 
 URLError = functools.partial(_make_exception, 'url_error')
+
+SubrepoMergeException = functools.partial(_make_exception, 'subrepo_merge_error')
+
+
+class HTTPRepoLocked(HTTPLocked):
+    """
+    Subclass of HTTPLocked response that allows to set the title and status
+    code via constructor arguments.
+    """
+    def __init__(self, title, status_code=None, **kwargs):
+        self.code = status_code or HTTPLocked.code
+        self.title = title
+        super(HTTPRepoLocked, self).__init__(**kwargs)

vcsserver/git.py

@@ -35,9 +35,9 b' from dulwich.server import update_server'
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
-    hg_url, httpbasicauthhandler, httpdigestauthhandler)
+    hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 
 
 DIR_STAT = stat.S_IFDIR
@@ -152,7 +152,7 b' class GitRemote(object):'
 
     def _build_opener(self, url):
         handlers = []
-        url_obj = hg_url(url)
+        url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
@@ -167,10 +167,12 b' class GitRemote(object):'
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
-        url_obj = hg_url(url)
+        url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****'
+        url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
+        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
@@ -184,12 +186,14 b' class GitRemote(object):'
         req = urllib2.Request(cu, None, {})
 
         try:
+            log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
-                raise Exception('Return Code is not 200')
+                raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
+            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
-            raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
+            raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
@@ -199,7 +203,7 b' class GitRemote(object):'
             # old style git can return some other format !
             pass
         else:
-            raise urllib2.URLError(
+            raise exceptions.URLError(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
@@ -327,7 +331,7 b' class GitRemote(object):'
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
-            url_obj = hg_url(url)
+            url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
@@ -521,7 +525,10 b' class GitRemote(object):'
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
-        return stdout
+        prefix = 'git version'
+        if stdout.startswith(prefix):
+            stdout = stdout[len(prefix):]
+        return stdout.strip()
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):

vcsserver/hg.py

@@ -28,13 +28,13 b' from mercurial import commands'
 from mercurial import unionrepo
 
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
-    archival, bin, clone, config as hgconfig, diffopts, hex, hg_url,
-    httpbasicauthhandler, httpdigestauthhandler, httppeer, localrepository,
-    match, memctx, exchange, memfilectx, nullrev, patch, peer, revrange, ui,
-    Abort, LookupError, RepoError, RepoLookupError, InterventionRequired,
-    RequirementError)
+    archival, bin, clone, config as hgconfig, diffopts, hex,
+    hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
+    httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev,
+    patch, peer, revrange, ui, Abort, LookupError, RepoError, RepoLookupError,
+    InterventionRequired, RequirementError)
 
 log = logging.getLogger(__name__)
 
@@ -142,6 +142,11 b' class HgRemote(object):'
         }
 
     @reraise_safe_exceptions
+    def discover_hg_version(self):
+        from mercurial import util
+        return util.version()
+
+    @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
@@ -316,16 +321,18 b' class HgRemote(object):'
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
-        log.info("Checking URL for remote cloning/import: %s", url)
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
-        url_obj = hg_url(url)
+        url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****'
+        url_obj.query = obfuscate_qs(url_obj.query)
+
         cleaned_uri = str(url_obj)
+        log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
@@ -346,12 +353,12 b' class HgRemote(object):'
         req = urllib2.Request(cu, None, {})
 
         try:
-            log.debug("Trying to open URL %s", url)
+            log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
-            log.warning("URL cannot be opened: %s", url, exc_info=True)
+            log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
@@ -362,15 +369,17 b' class HgRemote(object):'
             else:
                 # check for pure hg repos
                 log.debug(
-                    "Verifying if URL is a Mercurial repository: %s", url)
+                    "Verifying if URL is a Mercurial repository: %s",
+                    cleaned_uri)
                 httppeer(make_ui_from_config(config), url).lookup('tip')
         except Exception as e:
-            log.warning("URL is not a valid Mercurial repository: %s", url)
+            log.warning("URL is not a valid Mercurial repository: %s",
+                        cleaned_uri)
             raise exceptions.URLError(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
-        log.info("URL is a valid Mercurial repository: %s", url)
+        log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
@@ -683,6 +692,13 b' class HgRemote(object):'
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
+
+        # In case of sub repositories are used mercurial prompts the user in
+        # case of merge conflicts or different sub repository sources. By
+        # setting the interactive flag to `False` mercurial doesn't prompt the
+        # used but instead uses a default value.
+        repo.ui.setconfig('ui', 'interactive', False)
+
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions

vcsserver/hgcompat.py

@@ -35,6 +35,7 b' from mercurial import discovery'
 from mercurial import unionrepo
 from mercurial import localrepo
 from mercurial import merge as hg_merge
+from mercurial import subrepo
 
 from mercurial.commands import clone, nullid, pull
 from mercurial.context import memctx, memfilectx

vcsserver/hgpatches.py

@@ -58,3 +58,77 b' def _dynamic_capabilities_wrapper(lfprot'
         return calc_capabilities(repo, proto)
 
     return _dynamic_capabilities
+
+
+def patch_subrepo_type_mapping():
+    from collections import defaultdict
+    from hgcompat import subrepo
+    from exceptions import SubrepoMergeException
+
+    class NoOpSubrepo(subrepo.abstractsubrepo):
+
+        def __init__(self, ctx, path, *args, **kwargs):
+            """Initialize abstractsubrepo part
+
+            ``ctx`` is the context referring this subrepository in the
+            parent repository.
+
+            ``path`` is the path to this subrepository as seen from
+            innermost repository.
+            """
+            self.ui = ctx.repo().ui
+            self._ctx = ctx
+            self._path = path
+
+        def storeclean(self, path):
+            """
+            returns true if the repository has not changed since it was last
+            cloned from or pushed to a given repository.
+            """
+            return True
+
+        def dirty(self, ignoreupdate=False):
+            """returns true if the dirstate of the subrepo is dirty or does not
+            match current stored state. If ignoreupdate is true, only check
+            whether the subrepo has uncommitted changes in its dirstate.
+            """
+            return False
+
+        def basestate(self):
+            """current working directory base state, disregarding .hgsubstate
+            state and working directory modifications"""
+            substate = subrepo.state(self._ctx, self.ui)
+            file_system_path, rev, repotype = substate.get(self._path)
+            return rev
+
+        def remove(self):
+            """remove the subrepo
+
+            (should verify the dirstate is not dirty first)
+            """
+            pass
+
+        def get(self, state, overwrite=False):
+            """run whatever commands are needed to put the subrepo into
+            this state
+            """
+            pass
+
+        def merge(self, state):
+            """merge currently-saved state with the new state."""
+            raise SubrepoMergeException()
+
+        def push(self, opts):
+            """perform whatever action is analogous to 'hg push'
+
+            This may be a no-op on some systems.
+            """
+            pass
+
+    # Patch subrepo type mapping to always return our NoOpSubrepo class
+    # whenever a subrepo class is looked up.
+    subrepo.types = {
+        'hg': NoOpSubrepo,
+        'git': NoOpSubrepo,
+        'svn': NoOpSubrepo
+    }

vcsserver/http_main.py

@@ -31,6 +31,7 b' from pyramid.wsgi import wsgiapp'
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
+from vcsserver.exceptions import HTTPRepoLocked
 from vcsserver.server import VcsServer
 
 try:
@@ -181,6 +182,7 b' class HTTPApplication(object):'
             name='msgpack',
             factory=self._msgpack_renderer_factory)
 
+        self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
@@ -190,6 +192,9 b' class HTTPApplication(object):'
 
         self.config.add_view(
             self.status_view, route_name='status', renderer='json')
+        self.config.add_view(
+            self.service_view, route_name='service', renderer='msgpack')
+
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(
@@ -197,6 +202,9 b' class HTTPApplication(object):'
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
+        self.config.add_view(
+            self.handle_vcs_exception, context=Exception,
+            custom_predicates=[self.is_vcs_exception])
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
@@ -245,6 +253,19 b' class HTTPApplication(object):'
     def status_view(self, request):
         return {'status': 'OK'}
 
+    def service_view(self, request):
+        import vcsserver
+        payload = msgpack.unpackb(request.body, use_list=True)
+        resp = {
+            'id': payload.get('id'),
+            'result': dict(
+                version=vcsserver.__version__,
+                config={},
+                payload=payload,
+            )
+        }
+        return resp
+
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             value = msgpack.packb(value)
@@ -317,6 +338,23 b' class HTTPApplication(object):'
                 return app(environ, start_response)
             return _git_stream
 
+    def is_vcs_exception(self, context, request):
+        """
+        View predicate that returns true if the context object is a VCS
+        exception.
+        """
+        return hasattr(context, '_vcs_kind')
+
+    def handle_vcs_exception(self, exception, request):
+        if exception._vcs_kind == 'repo_locked':
+            # Get custom repo-locked status code if present.
+            status_code = request.headers.get('X-RC-Locked-Status-Code')
+            return HTTPRepoLocked(
+                title=exception.message, status_code=status_code)
+
+        # Re-raise exception if we can not handle it.
+        raise exception
+
 
 class ResponseFilter(object):
 
@@ -333,5 +371,6 b' class ResponseFilter(object):'
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
+        hgpatches.patch_subrepo_type_mapping()
     app = HTTPApplication(settings=settings)
     return app.wsgi_app()

vcsserver/main.py

@@ -503,5 +503,6 b' class VcsServerCommand(object):'
 def main(argv=sys.argv, quiet=False):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
+        hgpatches.patch_subrepo_type_mapping()
     command = VcsServerCommand(argv, quiet=quiet)
     return command.run()

vcsserver/svn.py

@@ -32,6 +32,7 b' import svn.fs'
 import svn.repos
 
 from vcsserver import svn_diff
+from vcsserver import exceptions
 from vcsserver.base import RepoFactory
 
 
@@ -48,6 +49,30 b' svn_compatible_versions = set(['
 ])
 
 
+def reraise_safe_exceptions(func):
+    """Decorator for converting svn exceptions to something neutral."""
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            if not hasattr(e, '_vcs_kind'):
+                log.exception("Unhandled exception in hg remote call")
+                raise_from_original(exceptions.UnhandledException)
+            raise
+    return wrapper
+
+
+def raise_from_original(new_type):
+    """
+    Raise a new exception type with original args and traceback.
+    """
+    _, original, traceback = sys.exc_info()
+    try:
+        raise new_type(*original.args), None, traceback
+    finally:
+        del traceback
+
+
 class SubversionFactory(RepoFactory):
 
     def _create_repo(self, wire, create, compatible_version):
@@ -88,6 +113,15 b' class SvnRemote(object):'
         # for subversion
         self._hg_factory = hg_factory
 
+    @reraise_safe_exceptions
+    def discover_svn_version(self):
+        try:
+            import svn.core
+            svn_ver = svn.core.SVN_VERSION
+        except ImportError:
+            svn_ver = None
+        return svn_ver
+
     def check_url(self, url, config_items):
         # this can throw exception if not installed, but we detect this
         from hgsubversion import svnrepo
@@ -163,13 +197,15 b' class SvnRemote(object):'
         for path, change in editor.changes.iteritems():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
+
             if change.item_kind == svn.core.svn_node_dir:
                 continue
-            if change.action == svn.repos.CHANGE_ACTION_ADD:
+            if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
-            elif change.action == svn.repos.CHANGE_ACTION_MODIFY:
+            elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
+                                   svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
-            elif change.action == svn.repos.CHANGE_ACTION_DELETE:
+            elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(