diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.4.2 +current_version = 4.5.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.4.2 +state = in_progress +version = 4.5.0 diff --git a/configs/production_http.ini b/configs/production_http.ini --- a/configs/production_http.ini +++ b/configs/production_http.ini @@ -3,22 +3,6 @@ # # ################################################################################ -[app:main] -use = egg:rhodecode-vcsserver - -pyramid.default_locale_name = en -pyramid.includes = - -# default locale used by VCS systems -locale = en_US.UTF-8 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true [server:main] ## COMMON ## @@ -29,7 +13,7 @@ port = 9900 ########################## ## GUNICORN WSGI SERVER ## ########################## -## run with gunicorn --log-config --paste +## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini use = egg:gunicorn#main ## Sets the number of process workers. You must set `instance_id = *` ## when this option is set to more than one worker, recommended @@ -52,6 +36,22 @@ max_requests_jitter = 30 timeout = 21600 +[app:main] +use = egg:rhodecode-vcsserver + +pyramid.default_locale_name = en +pyramid.includes = + +## default locale used by VCS systems +locale = en_US.UTF-8 + +# cache regions, please don't change +beaker.cache.regions = repo_object +beaker.cache.repo_object.type = memorylru +beaker.cache.repo_object.max_items = 100 +# cache auto-expires after N seconds +beaker.cache.repo_object.expire = 300 +beaker.cache.repo_object.enabled = true ################################ diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -16,12 +16,19 @@ let pkgs = pkgs_.overridePackages (self: super: { # Override subversion derivation to # - activate python bindings - # - set version to 1.8 - subversion = super.subversion18.override { - httpSupport = true; - pythonBindings = true; - python = self.python27Packages.python; - }; + subversion = let + subversionWithPython = super.subversion.override { + httpSupport = true; + pythonBindings = true; + python = self.python27Packages.python; + }; + in pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: { + patches = (oldAttrs.patches or []) ++ + pkgs.lib.optionals pkgs.stdenv.isDarwin [ + # johbo: "import svn.client" fails on darwin currently. + ./pkgs/subversion-1.9.4-darwin.patch + ]; + }); }); inherit (pkgs.lib) fix extends; @@ -86,21 +93,6 @@ let pythonPackages = self; }; - # Somewhat snappier setup of the development environment - # TODO: move into shell.nix - # TODO: think of supporting a stable path again, so that multiple shells - # can share it. - shellHook = '' - # Set locale - export LC_ALL="en_US.UTF-8" - - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${self.python.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${self.python.sitePackages} - python setup.py develop --prefix $tmp_path --allow-hosts "" - ''; - # Add VCSServer bin directory to path so that tests can find 'vcsserver'. preCheck = '' export PATH="$out/bin:$PATH" diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -13,7 +13,8 @@ in self: super: { subvertpy = super.subvertpy.override (attrs: { - SVN_PREFIX = "${pkgs.subversion}"; + # TODO: johbo: Remove the "or" once we drop 16.03 support + SVN_PREFIX = "${pkgs.subversion.dev or pkgs.subversion}"; propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ pkgs.aprutil pkgs.subversion diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1,3 +1,6 @@ +# Generated by pip2nix 0.4.0.dev1 +# See https://github.com/johbo/pip2nix + { Beaker = super.buildPythonPackage { name = "Beaker-1.7.0"; @@ -26,13 +29,13 @@ }; }; Mako = super.buildPythonPackage { - name = "Mako-1.0.4"; + name = "Mako-1.0.6"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [MarkupSafe]; src = fetchurl { - url = "https://pypi.python.org/packages/7a/ae/925434246ee90b42e8ef57d3b30a0ab7caf9a2de3e449b876c56dcb48155/Mako-1.0.4.tar.gz"; - md5 = "c5fc31a323dd4990683d2f2da02d4e20"; + url = "https://pypi.python.org/packages/56/4b/cb75836863a6382199aefb3d3809937e21fa4cb0db15a4f4ba0ecc2e7e8e/Mako-1.0.6.tar.gz"; + md5 = "a28e22a339080316b2acc352b9ee631c"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -103,6 +106,19 @@ license = [ pkgs.lib.licenses.mit ]; }; }; + backports.shutil-get-terminal-size = super.buildPythonPackage { + name = "backports.shutil-get-terminal-size-1.0.0"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz"; + md5 = "03267762480bd86b50580dc19dff3c66"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; configobj = super.buildPythonPackage { name = "configobj-5.0.6"; buildInputs = with self; []; @@ -116,6 +132,19 @@ license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; + decorator = super.buildPythonPackage { + name = "decorator-4.0.10"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz"; + md5 = "434b57fdc3230c500716c5aff8896100"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ]; + }; + }; dulwich = super.buildPythonPackage { name = "dulwich-0.13.0"; buildInputs = with self; []; @@ -129,6 +158,19 @@ license = [ pkgs.lib.licenses.gpl2Plus ]; }; }; + enum34 = super.buildPythonPackage { + name = "enum34-1.1.6"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; + md5 = "5f13a0841a61f7fc295c514490d120d0"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; greenlet = super.buildPythonPackage { name = "greenlet-0.4.7"; buildInputs = with self; []; @@ -181,6 +223,45 @@ license = [ pkgs.lib.licenses.zpt21 ]; }; }; + ipdb = super.buildPythonPackage { + name = "ipdb-0.10.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [ipython setuptools]; + src = fetchurl { + url = "https://pypi.python.org/packages/eb/0a/0a37dc19572580336ad3813792c0d18c8d7117c2d66fc63c501f13a7a8f8/ipdb-0.10.1.tar.gz"; + md5 = "4aeab65f633ddc98ebdb5eebf08dc713"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + ipython = super.buildPythonPackage { + name = "ipython-5.1.0"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [setuptools decorator pickleshare simplegeneric traitlets prompt-toolkit pygments pexpect backports.shutil-get-terminal-size pathlib2 pexpect]; + src = fetchurl { + url = "https://pypi.python.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz"; + md5 = "47c8122420f65b58784cb4b9b4af35e3"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + ipython-genutils = super.buildPythonPackage { + name = "ipython-genutils-0.1.0"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/71/b7/a64c71578521606edbbce15151358598f3dfb72a3431763edc2baf19e71f/ipython_genutils-0.1.0.tar.gz"; + md5 = "9a8afbe0978adbcbfcb3b35b2d015a56"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; mercurial = super.buildPythonPackage { name = "mercurial-3.8.4"; buildInputs = with self; []; @@ -220,6 +301,71 @@ license = [ pkgs.lib.licenses.asl20 ]; }; }; + pathlib2 = super.buildPythonPackage { + name = "pathlib2-2.1.0"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [six]; + src = fetchurl { + url = "https://pypi.python.org/packages/c9/27/8448b10d8440c08efeff0794adf7d0ed27adb98372c70c7b38f3947d4749/pathlib2-2.1.0.tar.gz"; + md5 = "38e4f58b4d69dfcb9edb49a54a8b28d2"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + pexpect = super.buildPythonPackage { + name = "pexpect-4.2.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [ptyprocess]; + src = fetchurl { + url = "https://pypi.python.org/packages/e8/13/d0b0599099d6cd23663043a2a0bb7c61e58c6ba359b2656e6fb000ef5b98/pexpect-4.2.1.tar.gz"; + md5 = "3694410001a99dff83f0b500a1ca1c95"; + }; + meta = { + license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ]; + }; + }; + pickleshare = super.buildPythonPackage { + name = "pickleshare-0.7.4"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [pathlib2]; + src = fetchurl { + url = "https://pypi.python.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz"; + md5 = "6a9e5dd8dfc023031f6b7b3f824cab12"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + prompt-toolkit = super.buildPythonPackage { + name = "prompt-toolkit-1.0.9"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [six wcwidth]; + src = fetchurl { + url = "https://pypi.python.org/packages/83/14/5ac258da6c530eca02852ee25c7a9ff3ca78287bb4c198d0d0055845d856/prompt_toolkit-1.0.9.tar.gz"; + md5 = "a39f91a54308fb7446b1a421c11f227c"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + ptyprocess = super.buildPythonPackage { + name = "ptyprocess-0.5.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/db/d7/b465161910f3d1cef593c5e002bff67e0384898f597f1a7fdc8db4c02bf6/ptyprocess-0.5.1.tar.gz"; + md5 = "94e537122914cc9ec9c1eadcd36e73a1"; + }; + meta = { + license = [ ]; + }; + }; py = super.buildPythonPackage { name = "py-1.4.29"; buildInputs = with self; []; @@ -233,6 +379,19 @@ license = [ pkgs.lib.licenses.mit ]; }; }; + pygments = super.buildPythonPackage { + name = "pygments-2.1.3"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/b8/67/ab177979be1c81bc99c8d0592ef22d547e70bb4c6815c383286ed5dec504/Pygments-2.1.3.tar.gz"; + md5 = "ed3fba2467c8afcda4d317e4ef2c6150"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; pyramid = super.buildPythonPackage { name = "pyramid-1.6.1"; buildInputs = with self; []; @@ -299,8 +458,8 @@ }; }; rhodecode-vcsserver = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.4.2"; - buildInputs = with self; [mock pytest WebTest]; + name = "rhodecode-vcsserver-4.5.0"; + buildInputs = with self; [mock pytest pytest-sugar WebTest]; doCheck = true; propagatedBuildInputs = with self; [configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid Pyro4 simplejson subprocess32 waitress WebOb]; src = ./.; @@ -334,6 +493,19 @@ license = [ pkgs.lib.licenses.mit ]; }; }; + simplegeneric = super.buildPythonPackage { + name = "simplegeneric-0.8.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"; + md5 = "f9c1fab00fd981be588fc32759f474e3"; + }; + meta = { + license = [ pkgs.lib.licenses.zpt21 ]; + }; + }; simplejson = super.buildPythonPackage { name = "simplejson-3.7.2"; buildInputs = with self; []; @@ -344,7 +516,7 @@ md5 = "a5fc7d05d4cb38492285553def5d4b46"; }; meta = { - license = [ pkgs.lib.licenses.mit pkgs.lib.licenses.afl21 ]; + license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ]; }; }; six = super.buildPythonPackage { @@ -386,6 +558,19 @@ license = [ pkgs.lib.licenses.lgpl21Plus ]; }; }; + traitlets = super.buildPythonPackage { + name = "traitlets-4.3.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [ipython-genutils six decorator enum34]; + src = fetchurl { + url = "https://pypi.python.org/packages/b1/d6/5b5aa6d5c474691909b91493da1e8972e309c9f01ecfe4aeafd272eb3234/traitlets-4.3.1.tar.gz"; + md5 = "dd0b1b6e5d31ce446d55a4b5e5083c98"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; translationstring = super.buildPythonPackage { name = "translationstring-1.3"; buildInputs = with self; []; @@ -425,6 +610,19 @@ license = [ pkgs.lib.licenses.zpt21 ]; }; }; + wcwidth = super.buildPythonPackage { + name = "wcwidth-0.1.7"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz"; + md5 = "b3b6a0a08f0c8a34d1de8cf44150a4ad"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; wheel = super.buildPythonPackage { name = "wheel-0.29.0"; buildInputs = with self; []; @@ -467,5 +665,30 @@ ### Test requirements - + pytest-sugar = super.buildPythonPackage { + name = "pytest-sugar-0.7.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [pytest termcolor]; + src = fetchurl { + url = "https://pypi.python.org/packages/03/97/05d988b4fa870e7373e8ee4582408543b9ca2bd35c3c67b569369c6f9c49/pytest-sugar-0.7.1.tar.gz"; + md5 = "7400f7c11f3d572b2c2a3b60352d35fe"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + termcolor = super.buildPythonPackage { + name = "termcolor-1.1.0"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; []; + src = fetchurl { + url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz"; + md5 = "043e89644f8909d462fbbfa511c768df"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; } diff --git a/pkgs/subversion-1.9.4-darwin.patch b/pkgs/subversion-1.9.4-darwin.patch new file mode 100644 --- /dev/null +++ b/pkgs/subversion-1.9.4-darwin.patch @@ -0,0 +1,63 @@ +diff -rup subversion-1.9.4-orig/subversion/include/svn_auth.h subversion-1.9.4/subversion/include/svn_auth.h +--- subversion-1.9.4-orig/subversion/include/svn_auth.h 2015-02-13 12:17:40.000000000 +0100 ++++ subversion-1.9.4/subversion/include/svn_auth.h 2016-09-21 12:55:27.000000000 +0200 +@@ -943,7 +943,7 @@ svn_auth_get_windows_ssl_server_trust_pr + + #endif /* WIN32 && !__MINGW32__ || DOXYGEN */ + +-#if defined(DARWIN) || defined(DOXYGEN) ++#if defined(SVN_HAVE_KEYCHAIN_SERVICES) || defined(DOXYGEN) + /** + * Set @a *provider to an authentication provider of type @c + * svn_auth_cred_simple_t that gets/sets information from the user's +@@ -984,7 +984,7 @@ void + svn_auth_get_keychain_ssl_client_cert_pw_provider( + svn_auth_provider_object_t **provider, + apr_pool_t *pool); +-#endif /* DARWIN || DOXYGEN */ ++#endif /* SVN_HAVE_KEYCHAIN_SERVICES || DOXYGEN */ + + /* Note that the gnome keyring unlock prompt related items below must be + * declared for all platforms in order to allow SWIG interfaces to be +diff -rup subversion-1.9.4-orig/subversion/libsvn_subr/auth.h subversion-1.9.4/subversion/libsvn_subr/auth.h +--- subversion-1.9.4-orig/subversion/libsvn_subr/auth.h 2015-08-27 06:00:31.000000000 +0200 ++++ subversion-1.9.4/subversion/libsvn_subr/auth.h 2016-09-21 12:56:20.000000000 +0200 +@@ -103,7 +103,7 @@ svn_auth__get_windows_ssl_server_trust_p + apr_pool_t *pool); + #endif /* WIN32 && !__MINGW32__ || DOXYGEN */ + +-#if defined(DARWIN) || defined(DOXYGEN) ++#if defined(SVN_HAVE_KEYCHAIN_SERVICES) || defined(DOXYGEN) + /** + * Set @a *provider to an authentication provider of type @c + * svn_auth_cred_simple_t that gets/sets information from the user's +@@ -134,7 +134,7 @@ void + svn_auth__get_keychain_ssl_client_cert_pw_provider( + svn_auth_provider_object_t **provider, + apr_pool_t *pool); +-#endif /* DARWIN || DOXYGEN */ ++#endif /* SVN_HAVE_KEYCHAIN_SERVICES || DOXYGEN */ + + #if !defined(WIN32) || defined(DOXYGEN) + /** +diff -rup subversion-1.9.4-orig/subversion/libsvn_subr/deprecated.c subversion-1.9.4/subversion/libsvn_subr/deprecated.c +--- subversion-1.9.4-orig/subversion/libsvn_subr/deprecated.c 2015-08-27 06:00:31.000000000 +0200 ++++ subversion-1.9.4/subversion/libsvn_subr/deprecated.c 2016-09-21 12:57:08.000000000 +0200 +@@ -1479,7 +1479,7 @@ svn_auth_get_windows_ssl_server_trust_pr + #endif /* WIN32 && !__MINGW32__ */ + + /*** From macos_keychain.c ***/ +-#if defined(DARWIN) ++#if defined(SVN_HAVE_KEYCHAIN_SERVICES) + void + svn_auth_get_keychain_simple_provider(svn_auth_provider_object_t **provider, + apr_pool_t *pool) +@@ -1494,7 +1494,7 @@ svn_auth_get_keychain_ssl_client_cert_pw + { + svn_auth__get_keychain_ssl_client_cert_pw_provider(provider, pool); + } +-#endif /* DARWIN */ ++#endif /* SVN_HAVE_KEYCHAIN_SERVICES */ + + #if !defined(WIN32) + void diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ configobj==5.0.6 dulwich==0.13.0 hgsubversion==1.8.6 infrae.cache==1.0.1 +ipdb==0.10.1 mercurial==3.8.4 msgpack-python==0.4.6 py==1.4.29 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ setup( tests_require=[ 'mock', 'pytest', + 'pytest-sugar', 'WebTest', ], install_requires=[ diff --git a/shell.nix b/shell.nix --- a/shell.nix +++ b/shell.nix @@ -1,18 +1,41 @@ { pkgs ? import {} -, doCheck ? false +, doCheck ? false }: let + vcsserver = import ./default.nix { - inherit - doCheck - pkgs; + inherit pkgs doCheck; }; + vcs-pythonPackages = vcsserver.pythonPackages; + in vcsserver.override (attrs: { # Avoid that we dump any sources into the store when entering the shell and # make development a little bit more convenient. src = null; + buildInputs = + attrs.buildInputs ++ + (with vcs-pythonPackages; [ + ipdb + ]); + + # Somewhat snappier setup of the development environment + # TODO: think of supporting a stable path again, so that multiple shells + # can share it. + postShellHook = '' + # Set locale + export LC_ALL="en_US.UTF-8" + + # Custom prompt to distinguish from other dev envs. + export PS1="\n\[\033[1;32m\][VCS-shell:\w]$\[\033[0m\] " + + tmp_path=$(mktemp -d) + export PATH="$tmp_path/bin:$PATH" + export PYTHONPATH="$tmp_path/${vcs-pythonPackages.python.sitePackages}:$PYTHONPATH" + mkdir -p $tmp_path/${vcs-pythonPackages.python.sitePackages} + python setup.py develop --prefix $tmp_path --allow-hosts "" + ''; }) diff --git a/tests/test_main.py b/tests/test_main.py --- a/tests/test_main.py +++ b/tests/test_main.py @@ -16,8 +16,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import mock +import pytest from vcsserver import main +from vcsserver.base import obfuscate_qs @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock()) @@ -34,3 +36,22 @@ def test_applies_largefiles_patch(patch_ mock.Mock(side_effect=Exception("Must not be called"))) def test_applies_largefiles_patch_only_if_mercurial_is_available(): main.main([]) + + +@pytest.mark.parametrize('given, expected', [ + ('bad', 'bad'), + ('query&foo=bar', 'query&foo=bar'), + ('equery&auth_token=bar', 'equery&auth_token=*****'), + ('a;b;c;query&foo=bar&auth_token=secret', + 'a&b&c&query&foo=bar&auth_token=*****'), + ('', ''), + (None, None), + ('foo=bar', 'foo=bar'), + ('auth_token=secret', 'auth_token=*****'), + ('auth_token=secret&api_key=secret2', + 'auth_token=*****&api_key=*****'), + ('auth_token=secret&api_key=secret2¶m=value', + 'auth_token=*****&api_key=*****¶m=value'), +]) +def test_obfuscate_qs(given, expected): + assert expected == obfuscate_qs(given) diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.4.2 \ No newline at end of file +4.5.0 \ No newline at end of file diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -16,7 +16,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging - +import urlparse log = logging.getLogger(__name__) @@ -69,3 +69,17 @@ class RepoFactory(object): 'INIT %s@%s repo object based on wire %s. Context: %s', self.__class__.__name__, wire['path'], wire, context) return createfunc() + + +def obfuscate_qs(query_string): + if query_string is None: + return None + + parsed = [] + for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True): + if k in ['auth_token', 'api_key']: + v = "*****" + parsed.append((k, v)) + + return '&'.join('{}{}'.format( + k, '={}'.format(v) if v else '') for k, v in parsed) diff --git a/vcsserver/exceptions.py b/vcsserver/exceptions.py --- a/vcsserver/exceptions.py +++ b/vcsserver/exceptions.py @@ -25,6 +25,7 @@ different error conditions. """ import functools +from pyramid.httpexceptions import HTTPLocked def _make_exception(kind, *args): @@ -54,3 +55,16 @@ RequirementException = functools.partial UnhandledException = functools.partial(_make_exception, 'unhandled') URLError = functools.partial(_make_exception, 'url_error') + +SubrepoMergeException = functools.partial(_make_exception, 'subrepo_merge_error') + + +class HTTPRepoLocked(HTTPLocked): + """ + Subclass of HTTPLocked response that allows to set the title and status + code via constructor arguments. + """ + def __init__(self, title, status_code=None, **kwargs): + self.code = status_code or HTTPLocked.code + self.title = title + super(HTTPRepoLocked, self).__init__(**kwargs) diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -35,9 +35,9 @@ from dulwich.server import update_server from vcsserver import exceptions, settings, subprocessio from vcsserver.utils import safe_str -from vcsserver.base import RepoFactory +from vcsserver.base import RepoFactory, obfuscate_qs from vcsserver.hgcompat import ( - hg_url, httpbasicauthhandler, httpdigestauthhandler) + hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler) DIR_STAT = stat.S_IFDIR @@ -152,7 +152,7 @@ class GitRemote(object): def _build_opener(self, url): handlers = [] - url_obj = hg_url(url) + url_obj = url_parser(url) _, authinfo = url_obj.authinfo() if authinfo: @@ -167,10 +167,12 @@ class GitRemote(object): @reraise_safe_exceptions def check_url(self, url, config): - url_obj = hg_url(url) + url_obj = url_parser(url) test_uri, _ = url_obj.authinfo() url_obj.passwd = '*****' + url_obj.query = obfuscate_qs(url_obj.query) cleaned_uri = str(url_obj) + log.info("Checking URL for remote cloning/import: %s", cleaned_uri) if not test_uri.endswith('info/refs'): test_uri = test_uri.rstrip('/') + '/info/refs' @@ -184,12 +186,14 @@ class GitRemote(object): req = urllib2.Request(cu, None, {}) try: + log.debug("Trying to open URL %s", cleaned_uri) resp = o.open(req) if resp.code != 200: - raise Exception('Return Code is not 200') + raise exceptions.URLError('Return Code is not 200') except Exception as e: + log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True) # means it cannot be cloned - raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e)) + raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e)) # now detect if it's proper git repo gitdata = resp.read() @@ -199,7 +203,7 @@ class GitRemote(object): # old style git can return some other format ! pass else: - raise urllib2.URLError( + raise exceptions.URLError( "url [%s] does not look like an git" % (cleaned_uri,)) return True @@ -327,7 +331,7 @@ class GitRemote(object): if url != 'default' and '://' not in url: client = LocalGitClient(url) else: - url_obj = hg_url(url) + url_obj = url_parser(url) o = self._build_opener(url) url, _ = url_obj.authinfo() client = HttpGitClient(base_url=url, opener=o) @@ -521,7 +525,10 @@ class GitRemote(object): def discover_git_version(self): stdout, _ = self.run_git_command( {}, ['--version'], _bare=True, _safe=True) - return stdout + prefix = 'git version' + if stdout.startswith(prefix): + stdout = stdout[len(prefix):] + return stdout.strip() @reraise_safe_exceptions def run_git_command(self, wire, cmd, **opts): diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -28,13 +28,13 @@ from mercurial import commands from mercurial import unionrepo from vcsserver import exceptions -from vcsserver.base import RepoFactory +from vcsserver.base import RepoFactory, obfuscate_qs from vcsserver.hgcompat import ( - archival, bin, clone, config as hgconfig, diffopts, hex, hg_url, - httpbasicauthhandler, httpdigestauthhandler, httppeer, localrepository, - match, memctx, exchange, memfilectx, nullrev, patch, peer, revrange, ui, - Abort, LookupError, RepoError, RepoLookupError, InterventionRequired, - RequirementError) + archival, bin, clone, config as hgconfig, diffopts, hex, + hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler, + httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev, + patch, peer, revrange, ui, Abort, LookupError, RepoError, RepoLookupError, + InterventionRequired, RequirementError) log = logging.getLogger(__name__) @@ -142,6 +142,11 @@ class HgRemote(object): } @reraise_safe_exceptions + def discover_hg_version(self): + from mercurial import util + return util.version() + + @reraise_safe_exceptions def archive_repo(self, archive_path, mtime, file_info, kind): if kind == "tgz": archiver = archival.tarit(archive_path, mtime, "gz") @@ -316,16 +321,18 @@ class HgRemote(object): @reraise_safe_exceptions def check_url(self, url, config): - log.info("Checking URL for remote cloning/import: %s", url) _proto = None if '+' in url[:url.find('://')]: _proto = url[0:url.find('+')] url = url[url.find('+') + 1:] handlers = [] - url_obj = hg_url(url) + url_obj = url_parser(url) test_uri, authinfo = url_obj.authinfo() url_obj.passwd = '*****' + url_obj.query = obfuscate_qs(url_obj.query) + cleaned_uri = str(url_obj) + log.info("Checking URL for remote cloning/import: %s", cleaned_uri) if authinfo: # create a password manager @@ -346,12 +353,12 @@ class HgRemote(object): req = urllib2.Request(cu, None, {}) try: - log.debug("Trying to open URL %s", url) + log.debug("Trying to open URL %s", cleaned_uri) resp = o.open(req) if resp.code != 200: raise exceptions.URLError('Return Code is not 200') except Exception as e: - log.warning("URL cannot be opened: %s", url, exc_info=True) + log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True) # means it cannot be cloned raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e)) @@ -362,15 +369,17 @@ class HgRemote(object): else: # check for pure hg repos log.debug( - "Verifying if URL is a Mercurial repository: %s", url) + "Verifying if URL is a Mercurial repository: %s", + cleaned_uri) httppeer(make_ui_from_config(config), url).lookup('tip') except Exception as e: - log.warning("URL is not a valid Mercurial repository: %s", url) + log.warning("URL is not a valid Mercurial repository: %s", + cleaned_uri) raise exceptions.URLError( "url [%s] does not look like an hg repo org_exc: %s" % (cleaned_uri, e)) - log.info("URL is a valid Mercurial repository: %s", url) + log.info("URL is a valid Mercurial repository: %s", cleaned_uri) return True @reraise_safe_exceptions @@ -683,6 +692,13 @@ class HgRemote(object): repo = self._factory.repo(wire) baseui = self._factory._create_config(wire['config']) repo.ui.setconfig('ui', 'merge', 'internal:dump') + + # In case of sub repositories are used mercurial prompts the user in + # case of merge conflicts or different sub repository sources. By + # setting the interactive flag to `False` mercurial doesn't prompt the + # used but instead uses a default value. + repo.ui.setconfig('ui', 'interactive', False) + commands.merge(baseui, repo, rev=revision) @reraise_safe_exceptions diff --git a/vcsserver/hgcompat.py b/vcsserver/hgcompat.py --- a/vcsserver/hgcompat.py +++ b/vcsserver/hgcompat.py @@ -35,6 +35,7 @@ from mercurial import discovery from mercurial import unionrepo from mercurial import localrepo from mercurial import merge as hg_merge +from mercurial import subrepo from mercurial.commands import clone, nullid, pull from mercurial.context import memctx, memfilectx diff --git a/vcsserver/hgpatches.py b/vcsserver/hgpatches.py --- a/vcsserver/hgpatches.py +++ b/vcsserver/hgpatches.py @@ -58,3 +58,77 @@ def _dynamic_capabilities_wrapper(lfprot return calc_capabilities(repo, proto) return _dynamic_capabilities + + +def patch_subrepo_type_mapping(): + from collections import defaultdict + from hgcompat import subrepo + from exceptions import SubrepoMergeException + + class NoOpSubrepo(subrepo.abstractsubrepo): + + def __init__(self, ctx, path, *args, **kwargs): + """Initialize abstractsubrepo part + + ``ctx`` is the context referring this subrepository in the + parent repository. + + ``path`` is the path to this subrepository as seen from + innermost repository. + """ + self.ui = ctx.repo().ui + self._ctx = ctx + self._path = path + + def storeclean(self, path): + """ + returns true if the repository has not changed since it was last + cloned from or pushed to a given repository. + """ + return True + + def dirty(self, ignoreupdate=False): + """returns true if the dirstate of the subrepo is dirty or does not + match current stored state. If ignoreupdate is true, only check + whether the subrepo has uncommitted changes in its dirstate. + """ + return False + + def basestate(self): + """current working directory base state, disregarding .hgsubstate + state and working directory modifications""" + substate = subrepo.state(self._ctx, self.ui) + file_system_path, rev, repotype = substate.get(self._path) + return rev + + def remove(self): + """remove the subrepo + + (should verify the dirstate is not dirty first) + """ + pass + + def get(self, state, overwrite=False): + """run whatever commands are needed to put the subrepo into + this state + """ + pass + + def merge(self, state): + """merge currently-saved state with the new state.""" + raise SubrepoMergeException() + + def push(self, opts): + """perform whatever action is analogous to 'hg push' + + This may be a no-op on some systems. + """ + pass + + # Patch subrepo type mapping to always return our NoOpSubrepo class + # whenever a subrepo class is looked up. + subrepo.types = { + 'hg': NoOpSubrepo, + 'git': NoOpSubrepo, + 'svn': NoOpSubrepo + } diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -31,6 +31,7 @@ from pyramid.wsgi import wsgiapp from vcsserver import remote_wsgi, scm_app, settings, hgpatches from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp +from vcsserver.exceptions import HTTPRepoLocked from vcsserver.server import VcsServer try: @@ -181,6 +182,7 @@ class HTTPApplication(object): name='msgpack', factory=self._msgpack_renderer_factory) + self.config.add_route('service', '/_service') self.config.add_route('status', '/status') self.config.add_route('hg_proxy', '/proxy/hg') self.config.add_route('git_proxy', '/proxy/git') @@ -190,6 +192,9 @@ class HTTPApplication(object): self.config.add_view( self.status_view, route_name='status', renderer='json') + self.config.add_view( + self.service_view, route_name='service', renderer='msgpack') + self.config.add_view(self.hg_proxy(), route_name='hg_proxy') self.config.add_view(self.git_proxy(), route_name='git_proxy') self.config.add_view( @@ -197,6 +202,9 @@ class HTTPApplication(object): self.config.add_view(self.hg_stream(), route_name='stream_hg') self.config.add_view(self.git_stream(), route_name='stream_git') + self.config.add_view( + self.handle_vcs_exception, context=Exception, + custom_predicates=[self.is_vcs_exception]) def wsgi_app(self): return self.config.make_wsgi_app() @@ -245,6 +253,19 @@ class HTTPApplication(object): def status_view(self, request): return {'status': 'OK'} + def service_view(self, request): + import vcsserver + payload = msgpack.unpackb(request.body, use_list=True) + resp = { + 'id': payload.get('id'), + 'result': dict( + version=vcsserver.__version__, + config={}, + payload=payload, + ) + } + return resp + def _msgpack_renderer_factory(self, info): def _render(value, system): value = msgpack.packb(value) @@ -317,6 +338,23 @@ class HTTPApplication(object): return app(environ, start_response) return _git_stream + def is_vcs_exception(self, context, request): + """ + View predicate that returns true if the context object is a VCS + exception. + """ + return hasattr(context, '_vcs_kind') + + def handle_vcs_exception(self, exception, request): + if exception._vcs_kind == 'repo_locked': + # Get custom repo-locked status code if present. + status_code = request.headers.get('X-RC-Locked-Status-Code') + return HTTPRepoLocked( + title=exception.message, status_code=status_code) + + # Re-raise exception if we can not handle it. + raise exception + class ResponseFilter(object): @@ -333,5 +371,6 @@ class ResponseFilter(object): def main(global_config, **settings): if MercurialFactory: hgpatches.patch_largefiles_capabilities() + hgpatches.patch_subrepo_type_mapping() app = HTTPApplication(settings=settings) return app.wsgi_app() diff --git a/vcsserver/main.py b/vcsserver/main.py --- a/vcsserver/main.py +++ b/vcsserver/main.py @@ -503,5 +503,6 @@ class VcsServerCommand(object): def main(argv=sys.argv, quiet=False): if MercurialFactory: hgpatches.patch_largefiles_capabilities() + hgpatches.patch_subrepo_type_mapping() command = VcsServerCommand(argv, quiet=quiet) return command.run() diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -32,6 +32,7 @@ import svn.fs import svn.repos from vcsserver import svn_diff +from vcsserver import exceptions from vcsserver.base import RepoFactory @@ -48,6 +49,30 @@ svn_compatible_versions = set([ ]) +def reraise_safe_exceptions(func): + """Decorator for converting svn exceptions to something neutral.""" + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + if not hasattr(e, '_vcs_kind'): + log.exception("Unhandled exception in hg remote call") + raise_from_original(exceptions.UnhandledException) + raise + return wrapper + + +def raise_from_original(new_type): + """ + Raise a new exception type with original args and traceback. + """ + _, original, traceback = sys.exc_info() + try: + raise new_type(*original.args), None, traceback + finally: + del traceback + + class SubversionFactory(RepoFactory): def _create_repo(self, wire, create, compatible_version): @@ -88,6 +113,15 @@ class SvnRemote(object): # for subversion self._hg_factory = hg_factory + @reraise_safe_exceptions + def discover_svn_version(self): + try: + import svn.core + svn_ver = svn.core.SVN_VERSION + except ImportError: + svn_ver = None + return svn_ver + def check_url(self, url, config_items): # this can throw exception if not installed, but we detect this from hgsubversion import svnrepo @@ -163,13 +197,15 @@ class SvnRemote(object): for path, change in editor.changes.iteritems(): # TODO: Decide what to do with directory nodes. Subversion can add # empty directories. + if change.item_kind == svn.core.svn_node_dir: continue - if change.action == svn.repos.CHANGE_ACTION_ADD: + if change.action in [svn.repos.CHANGE_ACTION_ADD]: added.append(path) - elif change.action == svn.repos.CHANGE_ACTION_MODIFY: + elif change.action in [svn.repos.CHANGE_ACTION_MODIFY, + svn.repos.CHANGE_ACTION_REPLACE]: changed.append(path) - elif change.action == svn.repos.CHANGE_ACTION_DELETE: + elif change.action in [svn.repos.CHANGE_ACTION_DELETE]: removed.append(path) else: raise NotImplementedError(