# HG changeset patch # User Marcin Kuzminski # Date 2018-11-02 15:00:53 # Node ID fd48aa4e4dbc9c9f7ede699c9de4b1e73ed304c9 # Parent d1b517587400fe591837389768eeb2846355c8d5 # Parent c2f54fe41a105d13e5da10eac0a9b1e713652927 release: Merge default into stable for release preparation diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.13.3 +current_version = 4.14.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.13.3 +state = in_progress +version = 4.14.0 diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -1,6 +1,5 @@ ################################################################################ # RhodeCode VCSServer with HTTP Backend - configuration # -# # ################################################################################ @@ -26,12 +25,13 @@ locale = en_US.UTF-8 ## at installation time, e.g /home/user/vcsserver-1/profile/bin core.binary_dir = "" -## custom exception store path, defaults to TMPDIR -exception_tracker.store_path = +## Custom exception store path, defaults to TMPDIR +## This is used to store exception from RhodeCode in shared directory +#exception_tracker.store_path = ## Default cache dir for caches. Putting this into a ramdisk ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require -## large ammount of space +## large amount of space cache_dir = %(here)s/rcdev/data ## cache region for storing repo_objects cache diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -1,6 +1,5 @@ ################################################################################ # RhodeCode VCSServer with HTTP Backend - configuration # -# # ################################################################################ @@ -47,12 +46,13 @@ locale = en_US.UTF-8 ## at installation time, e.g /home/user/vcsserver-1/profile/bin core.binary_dir = "" -## custom exception store path, defaults to TMPDIR -exception_tracker.store_path = +## Custom exception store path, defaults to TMPDIR +## This is used to store exception from RhodeCode in shared directory +#exception_tracker.store_path = ## Default cache dir for caches. Putting this into a ramdisk ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require -## large ammount of space +## large amount of space cache_dir = %(here)s/rcdev/data ## cache region for storing repo_objects cache diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -11,17 +11,17 @@ args@ , ... }: -let pkgs_ = (import {}); in +let + pkgs_ = (import {}); +in let - - # TODO: Currently we ignore the passed in pkgs, instead we should use it - # somehow as a base and apply overlays to it. pkgs = import { overlays = [ (import ./pkgs/overlays.nix) ]; - inherit (pkgs_) + inherit + (pkgs_) system; }; @@ -40,7 +40,7 @@ let in !builtins.elem (basename path) [ ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev" - "bower_components" "node_modules" + "node_modules" "node_binaries" "build" "data" "result" "tmp"] && !builtins.elem ext ["egg-info" "pyc"] && # TODO: johbo: This check is wrong, since "path" contains an absolute path, @@ -49,7 +49,11 @@ let sources = let - inherit (pkgs.lib) all isString attrValues; + inherit + (pkgs.lib) + all + isString + attrValues; sourcesConfig = pkgs.config.rc.sources or {}; in # Ensure that sources are configured as strings. Using a path @@ -121,27 +125,34 @@ let # python based programs need to be wrapped mkdir -p $out/bin - ln -s ${self.python}/bin/python $out/bin - ln -s ${self.pyramid}/bin/* $out/bin/ + ln -s ${self.python}/bin/python $out/bin/ ln -s ${self.gunicorn}/bin/gunicorn $out/bin/ + ln -s ${self.pyramid}/bin/prequest $out/bin/ + ln -s ${self.pyramid}/bin/pserve $out/bin/ # Symlink version control utilities # We ensure that always the correct version is available as a symlink. # So that users calling them via the profile path will always use the - # correct version. + # correct version. Wrapping is required so those can "import" + # vcsserver python hooks. ln -s ${pkgs.git}/bin/git $out/bin ln -s ${self.mercurial}/bin/hg $out/bin ln -s ${pkgs.subversion}/bin/svn* $out/bin + echo "DONE: created symlinks into $out/bin" + DEPS="$out/bin/*" - for file in $out/bin/*; + # wrap only dependency scripts, they require to have full PYTHONPATH set + # to be able to import all packages + for file in $DEPS; do wrapProgram $file \ --prefix PATH : $PATH \ --prefix PYTHONPATH : $PYTHONPATH \ --set PYTHONHASHSEED random done + echo "DONE: vcsserver binary wrapping" ''; @@ -156,12 +167,19 @@ let getAttr pythonPackages pkgs; pythonGeneratedPackages = import ./pkgs/python-packages.nix { - inherit pkgs; - inherit (pkgs) fetchurl fetchgit fetchhg; + inherit + pkgs; + inherit + (pkgs) + fetchurl + fetchgit + fetchhg; }; pythonVCSServerOverrides = import ./pkgs/python-packages-overrides.nix { - inherit pkgs basePythonPackages; + inherit + pkgs + basePythonPackages; }; diff --git a/pkgs/overlays.nix b/pkgs/overlays.nix --- a/pkgs/overlays.nix +++ b/pkgs/overlays.nix @@ -8,10 +8,10 @@ self: super: { }; patches = [ - ./git_patches/docbook2texi.patch - ./git_patches/symlinks-in-bin.patch - ./git_patches/git-sh-i18n.patch - ./git_patches/ssh-path.patch + ./patches/git/docbook2texi.patch + ./patches/git/symlinks-in-bin.patch + ./patches/git/git-sh-i18n.patch + ./patches/git/ssh-path.patch ]; }); diff --git a/pkgs/git_patches/docbook2texi.patch b/pkgs/patches/git/docbook2texi.patch rename from pkgs/git_patches/docbook2texi.patch rename to pkgs/patches/git/docbook2texi.patch diff --git a/pkgs/git_patches/git-sh-i18n.patch b/pkgs/patches/git/git-sh-i18n.patch rename from pkgs/git_patches/git-sh-i18n.patch rename to pkgs/patches/git/git-sh-i18n.patch diff --git a/pkgs/git_patches/ssh-path.patch b/pkgs/patches/git/ssh-path.patch rename from pkgs/git_patches/ssh-path.patch rename to pkgs/patches/git/ssh-path.patch diff --git a/pkgs/git_patches/symlinks-in-bin.patch b/pkgs/patches/git/symlinks-in-bin.patch rename from pkgs/git_patches/symlinks-in-bin.patch rename to pkgs/patches/git/symlinks-in-bin.patch diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -5,22 +5,22 @@ self: super: { "atomicwrites" = super.buildPythonPackage { - name = "atomicwrites-1.1.5"; + name = "atomicwrites-1.2.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/a1/e1/2d9bc76838e6e6667fde5814aa25d7feb93d6fa471bf6816daac2596e8b2/atomicwrites-1.1.5.tar.gz"; - sha256 = "11bm90fwm2avvf4f3ib8g925w7jr4m11vcsinn1bi6ns4bm32214"; + url = "https://files.pythonhosted.org/packages/ac/ed/a311712ef6b4355035489f665e63e1a73f9eb371929e3c98e5efd451069e/atomicwrites-1.2.1.tar.gz"; + sha256 = "1vmkbw9j0qammwxbxycrs39gvdg4lc2d4lk98kwf8ag2manyi6pc"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "attrs" = super.buildPythonPackage { - name = "attrs-18.1.0"; + name = "attrs-18.2.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/e4/ac/a04671e118b57bee87dabca1e0f2d3bda816b7a551036012d0ca24190e71/attrs-18.1.0.tar.gz"; - sha256 = "0yzqz8wv3w1srav5683a55v49i0szkm47dyrnkd56fqs8j8ypl70"; + url = "https://files.pythonhosted.org/packages/0f/9e/26b1d194aab960063b266170e53c39f73ea0d0d3f5ce23313e0ec8ee9bdf/attrs-18.2.0.tar.gz"; + sha256 = "0s9ydh058wmmf5v391pym877x4ahxg45dw6a0w4c7s5wgpigdjqh"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -77,14 +77,14 @@ self: super: { }; }; "coverage" = super.buildPythonPackage { - name = "coverage-3.7.1"; + name = "coverage-4.5.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz"; - sha256 = "0knlbq79g2ww6xzsyknj9rirrgrgc983dpa2d9nkdf31mb2a3bni"; + url = "https://files.pythonhosted.org/packages/35/fe/e7df7289d717426093c68d156e0fd9117c8f4872b6588e8a8928a0f68424/coverage-4.5.1.tar.gz"; + sha256 = "1wbrzpxka3xd4nmmkc6q0ir343d91kymwsm8pbmwa0d2a7q4ir2n"; }; meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; + license = [ pkgs.lib.licenses.asl20 ]; }; }; "decorator" = super.buildPythonPackage { @@ -99,11 +99,11 @@ self: super: { }; }; "dogpile.cache" = super.buildPythonPackage { - name = "dogpile.cache-0.6.6"; + name = "dogpile.cache-0.6.7"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/48/ca/604154d835c3668efb8a31bd979b0ea4bf39c2934a40ffecc0662296cb51/dogpile.cache-0.6.6.tar.gz"; - sha256 = "1h8n1lxd4l2qvahfkiinljkqz7pww7w3sgag0j8j9ixbl2h4wk84"; + url = "https://files.pythonhosted.org/packages/ee/bd/440da735a11c6087eed7cc8747fc4b995cbac2464168682f8ee1c8e43844/dogpile.cache-0.6.7.tar.gz"; + sha256 = "1aw8rx8vhb75y7zc6gi67g21sw057jdx7i8m3jq7kf3nqavxx9zw"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -154,14 +154,14 @@ self: super: { }; }; "gevent" = super.buildPythonPackage { - name = "gevent-1.3.5"; + name = "gevent-1.3.6"; doCheck = false; propagatedBuildInputs = [ self."greenlet" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/e6/0a/fc345c6e6161f84484870dbcaa58e427c10bd9bdcd08a69bed3d6b398bf1/gevent-1.3.5.tar.gz"; - sha256 = "1w3gydxirgd2f60c5yv579w4903ds9s4g3587ik4jby97hgqc5bz"; + url = "https://files.pythonhosted.org/packages/49/13/aa4bb3640b5167fe58875d3d7e65390cdb14f9682a41a741a566bb560842/gevent-1.3.6.tar.gz"; + sha256 = "1ih4k73dqz2zb561hda99vbanja3m6cdch3mgxxn1mla3qwkqhbv"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -179,11 +179,11 @@ self: super: { }; }; "greenlet" = super.buildPythonPackage { - name = "greenlet-0.4.13"; + name = "greenlet-0.4.15"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz"; - sha256 = "1r412gfx25jrdiv444prmz5a8igrfabwnwqyr6b52ypq7ga87vqg"; + url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz"; + sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -227,11 +227,11 @@ self: super: { }; }; "hupper" = super.buildPythonPackage { - name = "hupper-1.3"; + name = "hupper-1.3.1"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/51/0c/96335b1f2f32245fb871eea5bb9773196505ddb71fad15190056a282df9e/hupper-1.3.tar.gz"; - sha256 = "1pkyrm9c2crc32ps00k1ahnc5clj3pjwiarc7j0x8aykwih7ff10"; + url = "https://files.pythonhosted.org/packages/cf/4b/467b826a84c8594b81f414b5ab6794e981951dac90ca40abaf9ea1cb36b0/hupper-1.3.1.tar.gz"; + sha256 = "03mf13n6i4dd60wlb9m99ddl4m3lmly70cjp7f82vdkibfl1v6l9"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -371,15 +371,15 @@ self: super: { }; }; "pathlib2" = super.buildPythonPackage { - name = "pathlib2-2.3.0"; + name = "pathlib2-2.3.2"; doCheck = false; propagatedBuildInputs = [ self."six" self."scandir" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/a1/14/df0deb867c2733f7d857523c10942b3d6612a1b222502fdffa9439943dfb/pathlib2-2.3.0.tar.gz"; - sha256 = "1cx5gs2v9j2vnzmcrbq5l8fq2mwrr1h6pyf1sjdji2w1bavm09fk"; + url = "https://files.pythonhosted.org/packages/db/a8/7d6439c1aec525ed70810abee5b7d7f3aa35347f59bc28343e8f62019aa2/pathlib2-2.3.2.tar.gz"; + sha256 = "10yb0iv5x2hs631rcppkhbddx799d3h8pcwmkbh2a66ns3w71ccf"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -400,14 +400,14 @@ self: super: { }; }; "pickleshare" = super.buildPythonPackage { - name = "pickleshare-0.7.4"; + name = "pickleshare-0.7.5"; doCheck = false; propagatedBuildInputs = [ self."pathlib2" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz"; - sha256 = "0yvk14dzxk7g6qpr7iw23vzqbsr0dh4ij4xynkhnzpfz4xr2bac4"; + url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz"; + sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -443,11 +443,11 @@ self: super: { }; }; "pluggy" = super.buildPythonPackage { - name = "pluggy-0.6.0"; + name = "pluggy-0.8.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/11/bf/cbeb8cdfaffa9f2ea154a30ae31a9d04a1209312e2919138b4171a1f8199/pluggy-0.6.0.tar.gz"; - sha256 = "1zqckndfn85l1cd8pndw212zg1bq9fkg1nnj32kp2mppppsyg2kz"; + url = "https://files.pythonhosted.org/packages/65/25/81d0de17cd00f8ca994a4e74e3c4baf7cd25072c0b831dad5c7d9d6138f8/pluggy-0.8.0.tar.gz"; + sha256 = "1580p47l2zqzsza8jcnw1h2wh3vvmygk6ly8bvi4w0g8j14sjys4"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -469,11 +469,11 @@ self: super: { }; }; "psutil" = super.buildPythonPackage { - name = "psutil-5.4.6"; + name = "psutil-5.4.7"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/51/9e/0f8f5423ce28c9109807024f7bdde776ed0b1161de20b408875de7e030c3/psutil-5.4.6.tar.gz"; - sha256 = "1xmw4qi6hnrhw81xqzkvmsm9im7j2vkk4v26ycjwq2jczqsmlvk8"; + url = "https://files.pythonhosted.org/packages/7d/9a/1e93d41708f8ed2b564395edfa3389f0fd6d567597401c2e5e2775118d8b/psutil-5.4.7.tar.gz"; + sha256 = "0fsgmvzwbdbszkwfnqhib8jcxm4w6zyhvlxlcda0rfm5cyqj4qsv"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -491,11 +491,11 @@ self: super: { }; }; "py" = super.buildPythonPackage { - name = "py-1.5.3"; + name = "py-1.6.0"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/f7/84/b4c6e84672c4ceb94f727f3da8344037b62cee960d80e999b1cd9b832d83/py-1.5.3.tar.gz"; - sha256 = "10gq2lckvgwlk9w6yzijhzkarx44hsaknd0ypa08wlnpjnsgmj99"; + url = "https://files.pythonhosted.org/packages/4f/38/5f427d1eedae73063ce4da680d2bae72014995f9fdeaa57809df61c968cd/py-1.6.0.tar.gz"; + sha256 = "1wcs3zv9wl5m5x7p16avqj2gsrviyb23yvc3pr330isqs0sh98q6"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -552,7 +552,7 @@ self: super: { }; }; "pytest" = super.buildPythonPackage { - name = "pytest-3.6.0"; + name = "pytest-3.8.2"; doCheck = false; propagatedBuildInputs = [ self."py" @@ -563,25 +563,26 @@ self: super: { self."atomicwrites" self."pluggy" self."funcsigs" + self."pathlib2" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/67/6a/5bcdc22f8dbada1d2910d6e1a3a03f6b14306c78f81122890735b28be4bf/pytest-3.6.0.tar.gz"; - sha256 = "0bdfazvjjbxssqzyvkb3m2x2in7xv56ipr899l00s87k7815sm9r"; + url = "https://files.pythonhosted.org/packages/5f/d2/7f77f406ac505abda02ab4afb50d06ebf304f6ea42fca34f8f37529106b2/pytest-3.8.2.tar.gz"; + sha256 = "18nrwzn61kph2y6gxwfz9ms68rfvr9d4vcffsxng9p7jk9z18clk"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; "pytest-cov" = super.buildPythonPackage { - name = "pytest-cov-2.5.1"; + name = "pytest-cov-2.6.0"; doCheck = false; propagatedBuildInputs = [ self."pytest" self."coverage" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/24/b4/7290d65b2f3633db51393bdf8ae66309b37620bc3ec116c5e357e3e37238/pytest-cov-2.5.1.tar.gz"; - sha256 = "0bbfpwdh9k3636bxc88vz9fa7vf4akchgn513ql1vd0xy4n7bah3"; + url = "https://files.pythonhosted.org/packages/d9/e2/58f90a316fbd94dd50bf5c826a23f3f5d079fb3cc448c1e9f0e3c33a3d2a/pytest-cov-2.6.0.tar.gz"; + sha256 = "0qnpp9y3ygx4jk4pf5ad71fh2skbvnr6gl54m7rg5qysnx4g0q73"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ]; @@ -630,14 +631,14 @@ self: super: { }; }; "pytest-timeout" = super.buildPythonPackage { - name = "pytest-timeout-1.2.1"; + name = "pytest-timeout-1.3.2"; doCheck = false; propagatedBuildInputs = [ self."pytest" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/be/e9/a9106b8bc87521c6813060f50f7d1fdc15665bc1bbbe71c0ffc1c571aaa2/pytest-timeout-1.2.1.tar.gz"; - sha256 = "1kdp6qbh5v1168l99rba5yfzvy05gmzkmkhldgp36p9xcdjd5dv8"; + url = "https://files.pythonhosted.org/packages/8c/3e/1b6a319d12ae7baa3acb7c18ff2c8630a09471a0319d43535c683b4d03eb/pytest-timeout-1.3.2.tar.gz"; + sha256 = "09wnmzvnls2mnsdz7x3c3sk2zdp6jl4dryvyj5i8hqz16q2zq5qi"; }; meta = { license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ]; @@ -655,7 +656,7 @@ self: super: { }; }; "rhodecode-vcsserver" = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.13.3"; + name = "rhodecode-vcsserver-4.14.0"; buildInputs = [ self."pytest" self."py" @@ -663,8 +664,8 @@ self: super: { self."pytest-sugar" self."pytest-runner" self."pytest-profiling" + self."pytest-timeout" self."gprof2dot" - self."pytest-timeout" self."mock" self."webtest" self."cov-core" @@ -695,7 +696,6 @@ self: super: { self."repoze.lru" self."simplejson" self."subprocess32" - self."setproctitle" self."subvertpy" self."six" self."translationstring" @@ -706,6 +706,7 @@ self: super: { self."greenlet" self."gunicorn" self."waitress" + self."setproctitle" self."ipdb" self."ipython" self."pytest" @@ -714,8 +715,8 @@ self: super: { self."pytest-sugar" self."pytest-runner" self."pytest-profiling" + self."pytest-timeout" self."gprof2dot" - self."pytest-timeout" self."mock" self."webtest" self."cov-core" @@ -749,11 +750,11 @@ self: super: { }; }; "setuptools" = super.buildPythonPackage { - name = "setuptools-40.1.0"; + name = "setuptools-40.4.3"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/5a/df/b2e3d9693bb0dcbeac516a73dd7a9eb82b126ae52e4a74605a9b01beddd5/setuptools-40.1.0.zip"; - sha256 = "0w1blx5ajga5y15dci0mddk49cf2xpq0mp7rp7jrqr2diqk00ib6"; + url = "https://files.pythonhosted.org/packages/6e/9c/6a003320b00ef237f94aa74e4ad66c57a7618f6c79d67527136e2544b728/setuptools-40.4.3.zip"; + sha256 = "058v6zns4634n4al2nmmvp15j8nrgwn8wjrbdks47wk3vm05gg5c"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -793,11 +794,11 @@ self: super: { }; }; "subprocess32" = super.buildPythonPackage { - name = "subprocess32-3.5.1"; + name = "subprocess32-3.5.2"; doCheck = false; src = fetchurl { - url = "https://files.pythonhosted.org/packages/de/fb/fd3e91507021e2aecdb081d1b920082628d6b8869ead845e3e87b3d2e2ca/subprocess32-3.5.1.tar.gz"; - sha256 = "0wgi3bfnssid1g6h0v803z3k1wjal6il16nr3r9c587cfzwfkv0q"; + url = "https://files.pythonhosted.org/packages/c3/5f/7117737fc7114061837a4f51670d863dd7f7f9c762a6546fa8a0dcfe61c8/subprocess32-3.5.2.tar.gz"; + sha256 = "11v62shwmdys48g7ncs3a8jwwnkcl8d4zcwy6dk73z1zy2f9hazb"; }; meta = { license = [ pkgs.lib.licenses.psfl ]; diff --git a/release.nix b/release.nix --- a/release.nix +++ b/release.nix @@ -1,14 +1,12 @@ # This file defines how to "build" for packaging. -{ pkgs ? import {} -, doCheck ? true +{ doCheck ? true }: let vcsserver = import ./default.nix { inherit - doCheck - pkgs; + doCheck; }; in { diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ # our custom configobj https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c#egg=configobj==5.0.6 -atomicwrites==1.1.5 -attrs==18.1.0 -dogpile.cache==0.6.6 +atomicwrites==1.2.1 +attrs==18.2.0 +dogpile.cache==0.6.7 dogpile.core==0.4.1 decorator==4.1.2 dulwich==0.13.0 @@ -16,16 +16,15 @@ mercurial==4.6.2 msgpack-python==0.5.6 pastedeploy==1.5.2 -psutil==5.4.6 +psutil==5.4.7 pyramid==1.9.2 pyramid-mako==1.0.2 pygments==2.2.0 -pathlib2==2.3.0 +pathlib2==2.3.2 repoze.lru==0.7 simplejson==3.11.1 -subprocess32==3.5.1 -setproctitle==1.1.10 +subprocess32==3.5.2 subvertpy==0.10.1 six==1.11.0 @@ -35,10 +34,11 @@ zope.deprecation==4.3.0 zope.interface==4.5.0 ## http servers -gevent==1.3.5 -greenlet==0.4.13 +gevent==1.3.6 +greenlet==0.4.15 gunicorn==19.9.0 waitress==1.1.0 +setproctitle==1.1.10 ## debug ipdb==0.11.0 diff --git a/requirements_test.txt b/requirements_test.txt --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,14 +1,14 @@ # test related requirements -pytest==3.6.0 -py==1.5.3 -pytest-cov==2.5.1 +pytest==3.8.2 +py==1.6.0 +pytest-cov==2.6.0 pytest-sugar==0.9.1 pytest-runner==4.2.0 pytest-profiling==1.3.0 +pytest-timeout==1.3.2 gprof2dot==2017.9.19 -pytest-timeout==1.2.1 mock==1.0.1 webtest==2.0.29 cov-core==1.15.0 -coverage==3.7.1 +coverage==4.5.1 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -131,9 +131,6 @@ setup( 'Programming Language :: Python :: 2.7', ], entry_points={ - 'console_scripts': [ - 'vcsserver=vcsserver.main:main', - ], 'paste.app_factory': ['main=vcsserver.http_main:main'] }, ) diff --git a/shell.nix b/shell.nix --- a/shell.nix +++ b/shell.nix @@ -10,7 +10,6 @@ let vcsserver = import ./default.nix { inherit - pkgs doCheck; }; diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.13.3 \ No newline at end of file +4.14.0 \ No newline at end of file diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - +import collections import logging import os import posixpath as vcspath @@ -56,9 +56,11 @@ def reraise_safe_exceptions(func): return func(*args, **kwargs) except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing) as e: - raise exceptions.LookupException(e)(e.message) + exc = exceptions.LookupException(e) + raise exc(e) except (HangupException, UnexpectedCommandError) as e: - raise exceptions.VcsException(e)(e.message) + exc = exceptions.VcsException(e) + raise exc(e) except Exception as e: # NOTE(marcink): becuase of how dulwich handles some exceptions # (KeyError on empty repos), we cannot track this and catch all @@ -98,7 +100,7 @@ class GitRemote(object): def __init__(self, factory): self._factory = factory - + self.peeled_ref_marker = '^{}' self._bulk_methods = { "author": self.commit_attribute, "date": self.get_object_attrs, @@ -279,7 +281,8 @@ class GitRemote(object): @reraise_safe_exceptions def clone(self, wire, url, deferred, valid_refs, update_after_clone): - remote_refs = self.fetch(wire, url, apply_refs=False) + # TODO(marcink): deprecate this method. Last i checked we don't use it anymore + remote_refs = self.pull(wire, url, apply_refs=False) repo = self._factory.repo(wire) if isinstance(valid_refs, list): valid_refs = tuple(valid_refs) @@ -396,7 +399,7 @@ class GitRemote(object): return commit.id @reraise_safe_exceptions - def fetch(self, wire, url, apply_refs=True, refs=None): + def pull(self, wire, url, apply_refs=True, refs=None, update_after=False): if url != 'default' and '://' not in url: client = LocalGitClient(url) else: @@ -431,30 +434,78 @@ class GitRemote(object): # TODO: johbo: Needs proper test coverage with a git repository # that contains a tag object, so that we would end up with # a peeled ref at this point. - PEELED_REF_MARKER = '^{}' for k in remote_refs: - if k.endswith(PEELED_REF_MARKER): - log.info("Skipping peeled reference %s", k) + if k.endswith(self.peeled_ref_marker): + log.debug("Skipping peeled reference %s", k) continue repo[k] = remote_refs[k] - if refs: + if refs and not update_after: # mikhail: explicitly set the head to the last ref. repo['HEAD'] = remote_refs[refs[-1]] - # TODO: mikhail: should we return remote_refs here to be - # consistent? - else: - return remote_refs + if update_after: + # we want to checkout HEAD + repo["HEAD"] = remote_refs["HEAD"] + index.build_index_from_tree(repo.path, repo.index_path(), + repo.object_store, repo["HEAD"].tree) + return remote_refs + + @reraise_safe_exceptions + def sync_fetch(self, wire, url, refs=None): + repo = self._factory.repo(wire) + if refs and not isinstance(refs, (list, tuple)): + refs = [refs] + + # get all remote refs we'll use to fetch later + output, __ = self.run_git_command( + wire, ['ls-remote', url], fail_on_stderr=False, + _copts=['-c', 'core.askpass=""'], + extra_env={'GIT_TERMINAL_PROMPT': '0'}) + + remote_refs = collections.OrderedDict() + fetch_refs = [] + + for ref_line in output.splitlines(): + sha, ref = ref_line.split('\t') + sha = sha.strip() + if ref in remote_refs: + # duplicate, skip + continue + if ref.endswith(self.peeled_ref_marker): + log.debug("Skipping peeled reference %s", ref) + continue + # don't sync HEAD + if ref in ['HEAD']: + continue + + remote_refs[ref] = sha + + if refs and sha in refs: + # we filter fetch using our specified refs + fetch_refs.append('{}:{}'.format(ref, ref)) + elif not refs: + fetch_refs.append('{}:{}'.format(ref, ref)) + + if fetch_refs: + _out, _err = self.run_git_command( + wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs, + fail_on_stderr=False, + _copts=['-c', 'core.askpass=""'], + extra_env={'GIT_TERMINAL_PROMPT': '0'}) + + return remote_refs @reraise_safe_exceptions def sync_push(self, wire, url, refs=None): - if self.check_url(url, wire): - repo = self._factory.repo(wire) - self.run_git_command( - wire, ['push', url, '--mirror'], fail_on_stderr=False, - _copts=['-c', 'core.askpass=""'], - extra_env={'GIT_TERMINAL_PROMPT': '0'}) + if not self.check_url(url, wire): + return + + repo = self._factory.repo(wire) + self.run_git_command( + wire, ['push', url, '--mirror'], fail_on_stderr=False, + _copts=['-c', 'core.askpass=""'], + extra_env={'GIT_TERMINAL_PROMPT': '0'}) @reraise_safe_exceptions def get_remote_refs(self, wire, url): @@ -644,9 +695,9 @@ class GitRemote(object): gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1' cmd = [settings.GIT_EXECUTABLE] + _copts + cmd + _opts = {'env': gitenv, 'shell': False} try: - _opts = {'env': gitenv, 'shell': False} _opts.update(opts) p = subprocessio.SubprocessIOChunker(cmd, **_opts) @@ -654,7 +705,9 @@ class GitRemote(object): except (EnvironmentError, OSError) as err: cmd = ' '.join(cmd) # human friendly CMD tb_err = ("Couldn't run git command (%s).\n" - "Original error was:%s\n" % (cmd, err)) + "Original error was:%s\n" + "Call options:%s\n" + % (cmd, err, _opts)) log.exception(tb_err) if safe_call: return '', err diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -585,19 +585,21 @@ class HgRemote(object): @reraise_safe_exceptions def sync_push(self, wire, url): - if self.check_url(url, wire['config']): - repo = self._factory.repo(wire) + if not self.check_url(url, wire['config']): + return - # Disable any prompts for this repo - repo.ui.setconfig('ui', 'interactive', 'off', '-y') + repo = self._factory.repo(wire) + + # Disable any prompts for this repo + repo.ui.setconfig('ui', 'interactive', 'off', '-y') - bookmarks = dict(repo._bookmarks).keys() - remote = peer(repo, {}, url) - # Disable any prompts for this remote - remote.ui.setconfig('ui', 'interactive', 'off', '-y') + bookmarks = dict(repo._bookmarks).keys() + remote = peer(repo, {}, url) + # Disable any prompts for this remote + remote.ui.setconfig('ui', 'interactive', 'off', '-y') - return exchange.push( - repo, remote, newbranch=True, bookmarks=bookmarks).cgresult + return exchange.push( + repo, remote, newbranch=True, bookmarks=bookmarks).cgresult @reraise_safe_exceptions def revision(self, wire, rev): diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -214,6 +214,30 @@ def _check_heads(repo, start, end, commi return [] +def _get_git_env(): + env = {} + for k, v in os.environ.items(): + if k.startswith('GIT'): + env[k] = v + + # serialized version + return [(k, v) for k, v in env.items()] + + +def _get_hg_env(old_rev, new_rev, txnid, repo_path): + env = {} + for k, v in os.environ.items(): + if k.startswith('HG'): + env[k] = v + + env['HG_NODE'] = old_rev + env['HG_NODE_LAST'] = new_rev + env['HG_TXNID'] = txnid + env['HG_PENDING'] = repo_path + + return [(k, v) for k, v in env.items()] + + def repo_size(ui, repo, **kwargs): extras = _extras_from_ui(ui) return _call_hook('repo_size', extras, HgMessageWriter(ui)) @@ -260,6 +284,7 @@ def pre_push(ui, repo, node=None, **kwar for branch, commits in branches.items(): old_rev = kwargs.get('node_last') or commits[0] rev_data.append({ + 'total_commits': len(commits), 'old_rev': old_rev, 'new_rev': commits[-1], 'ref': '', @@ -270,7 +295,16 @@ def pre_push(ui, repo, node=None, **kwar for push_ref in rev_data: push_ref['multiple_heads'] = _heads + repo_path = os.path.join( + extras.get('repo_store', ''), extras.get('repository', '')) + push_ref['hg_env'] = _get_hg_env( + old_rev=push_ref['old_rev'], + new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'), + repo_path=repo_path) + + extras['hook_type'] = kwargs.get('hooktype', 'pre_push') extras['commit_ids'] = rev_data + return _call_hook('pre_push', extras, HgMessageWriter(ui)) @@ -319,6 +353,7 @@ def post_push(ui, repo, node, **kwargs): if hasattr(ui, '_rc_pushkey_branches'): bookmarks = ui._rc_pushkey_branches + extras['hook_type'] = kwargs.get('hooktype', 'post_push') extras['commit_ids'] = commit_ids extras['new_refs'] = { 'branches': branches, @@ -426,6 +461,10 @@ def _parse_git_ref_lines(revision_lines) ref_data = ref.split('/', 2) if ref_data[1] in ('tags', 'heads'): rev_data.append({ + # NOTE(marcink): + # we're unable to tell total_commits for git at this point + # but we set the variable for consistency with GIT + 'total_commits': -1, 'old_rev': old_rev, 'new_rev': new_rev, 'ref': ref, @@ -455,8 +494,7 @@ def git_pre_receive(unused_repo_path, re for push_ref in rev_data: # store our git-env which holds the temp store - push_ref['git_env'] = [ - (k, v) for k, v in os.environ.items() if k.startswith('GIT')] + push_ref['git_env'] = _get_git_env() push_ref['pruned_sha'] = '' if not detect_force_push: # don't check for forced-push when we don't need to @@ -476,6 +514,7 @@ def git_pre_receive(unused_repo_path, re if stdout: push_ref['pruned_sha'] = stdout.splitlines() + extras['hook_type'] = 'pre_receive' extras['commit_ids'] = rev_data return _call_hook('pre_push', extras, GitMessageWriter()) @@ -555,6 +594,7 @@ def git_post_receive(unused_repo_path, r tags.append(push_ref['name']) git_revs.append('tag=>%s' % push_ref['name']) + extras['hook_type'] = 'post_receive' extras['commit_ids'] = git_revs extras['new_refs'] = { 'branches': branches, @@ -586,6 +626,21 @@ def _get_extras_from_txn_id(path, txn_id return extras +def _get_extras_from_commit_id(commit_id, path): + extras = {} + try: + cmd = ['svnlook', 'pget', + '-r', commit_id, + '--revprop', path, 'rc-scm-extras'] + stdout, stderr = subprocessio.run_command( + cmd, env=os.environ.copy()) + extras = json.loads(base64.urlsafe_b64decode(stdout)) + except Exception: + log.exception('Failed to extract extras info from commit_id') + + return extras + + def svn_pre_commit(repo_path, commit_data, env): path, txn_id = commit_data branches = [] @@ -602,6 +657,7 @@ def svn_pre_commit(repo_path, commit_dat extras['commit_ids'] = [] extras['txn_id'] = txn_id extras['new_refs'] = { + 'total_commits': 1, 'branches': branches, 'bookmarks': [], 'tags': tags, @@ -610,21 +666,6 @@ def svn_pre_commit(repo_path, commit_dat return _call_hook('pre_push', extras, SvnMessageWriter()) -def _get_extras_from_commit_id(commit_id, path): - extras = {} - try: - cmd = ['svnlook', 'pget', - '-r', commit_id, - '--revprop', path, 'rc-scm-extras'] - stdout, stderr = subprocessio.run_command( - cmd, env=os.environ.copy()) - extras = json.loads(base64.urlsafe_b64decode(stdout)) - except Exception: - log.exception('Failed to extract extras info from commit_id') - - return extras - - def svn_post_commit(repo_path, commit_data, env): """ commit_data is path, rev, txn_id @@ -647,6 +688,7 @@ def svn_post_commit(repo_path, commit_da 'branches': branches, 'bookmarks': [], 'tags': tags, + 'total_commits': 1, } if 'repo_size' in extras['hooks']: diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -144,7 +144,7 @@ class VCS(object): def _configure_locale(self): if self.locale: - log.info('Settings locale: `LC_ALL` to %s' % self.locale) + log.info('Settings locale: `LC_ALL` to %s', self.locale) else: log.info( 'Configuring locale subsystem based on environment variables') diff --git a/vcsserver/lib/memory_lru_dict.py b/vcsserver/lib/memory_lru_dict.py --- a/vcsserver/lib/memory_lru_dict.py +++ b/vcsserver/lib/memory_lru_dict.py @@ -58,7 +58,7 @@ class LRUDictDebug(LRUDict): fmt = '\n' for cnt, elem in enumerate(self.keys()): fmt += '%s - %s\n' % (cnt+1, safe_str(elem)) - log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt)) + log.debug('current LRU keys (%s):%s', elems_cnt, fmt) def __getitem__(self, key): self._report_keys() diff --git a/vcsserver/tests/fixture.py b/vcsserver/tests/fixture.py --- a/vcsserver/tests/fixture.py +++ b/vcsserver/tests/fixture.py @@ -77,7 +77,7 @@ def no_newline_id_generator(test_name): nicer output of progress of test """ org_name = test_name - test_name = test_name\ + test_name = str(test_name)\ .replace('\n', '_N') \ .replace('\r', '_N') \ .replace('\t', '_T') \ diff --git a/vcsserver/tests/test_git.py b/vcsserver/tests/test_git.py --- a/vcsserver/tests/test_git.py +++ b/vcsserver/tests/test_git.py @@ -61,7 +61,7 @@ class TestGitFetch(object): with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch: mock_fetch.side_effect = side_effect - self.remote_git.fetch(wire=None, url='/tmp/', apply_refs=False) + self.remote_git.pull(wire=None, url='/tmp/', apply_refs=False) determine_wants = self.mock_repo.object_store.determine_wants_all determine_wants.assert_called_once_with(SAMPLE_REFS) @@ -78,7 +78,7 @@ class TestGitFetch(object): with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch: mock_fetch.side_effect = side_effect - self.remote_git.fetch( + self.remote_git.pull( wire=None, url='/tmp/', apply_refs=False, refs=selected_refs.keys()) determine_wants = self.mock_repo.object_store.determine_wants_all diff --git a/vcsserver/tests/test_hooks.py b/vcsserver/tests/test_hooks.py --- a/vcsserver/tests/test_hooks.py +++ b/vcsserver/tests/test_hooks.py @@ -70,7 +70,7 @@ def test_git_post_receive_calls_repo_siz with mock.patch.object(hooks, '_call_hook') as call_hook_mock: hooks.git_post_receive( None, '', {'RC_SCM_DATA': json.dumps(extras)}) - extras.update({'commit_ids': [], + extras.update({'commit_ids': [], 'hook_type': 'post_receive', 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}}) expected_calls = [ mock.call('repo_size', extras, mock.ANY), @@ -84,7 +84,7 @@ def test_git_post_receive_does_not_call_ with mock.patch.object(hooks, '_call_hook') as call_hook_mock: hooks.git_post_receive( None, '', {'RC_SCM_DATA': json.dumps(extras)}) - extras.update({'commit_ids': [], + extras.update({'commit_ids': [], 'hook_type': 'post_receive', 'new_refs': {'bookmarks': [], 'branches': [], 'tags': []}}) expected_calls = [ mock.call('post_push', extras, mock.ANY) diff --git a/vcsserver/tweens.py b/vcsserver/tweens.py --- a/vcsserver/tweens.py +++ b/vcsserver/tweens.py @@ -46,10 +46,8 @@ class RequestWrapperTween(object): finally: end = time.time() - log.info('IP: %s Request to path: `%s` time: %.3fs' % ( - '127.0.0.1', - safe_str(get_access_path(request)), end - start) - ) + log.info('IP: %s Request to path: `%s` time: %.3fs', + '127.0.0.1', safe_str(get_access_path(request)), end - start) return response