diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.12.4 +current_version = 4.13.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.12.4 +state = in_progress +version = 4.13.0 diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -15,4 +15,4 @@ test-clean: find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';' test-only: - PYTHONHASHSEED=random py.test -vv -r xw --cov=vcsserver --cov-report=term-missing --cov-report=html vcsserver + PYTHONHASHSEED=random py.test -vv -r xw -p no:sugar --cov=vcsserver --cov-report=term-missing --cov-report=html vcsserver diff --git a/configs/development.ini b/configs/development.ini old mode 120000 new mode 100644 --- a/configs/development.ini +++ b/configs/development.ini @@ -1,1 +1,79 @@ -development_http.ini \ No newline at end of file +################################################################################ +# RhodeCode VCSServer with HTTP Backend - configuration # +# # +################################################################################ + + +[server:main] +## COMMON ## +host = 0.0.0.0 +port = 9900 + +use = egg:waitress#main + + +[app:main] +use = egg:rhodecode-vcsserver + +pyramid.default_locale_name = en +pyramid.includes = + +## default locale used by VCS systems +locale = en_US.UTF-8 + + +## path to binaries for vcsserver, it should be set by the installer +## at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + +## cache region for storing repo_objects cache +rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru +## cache auto-expires after N seconds +rc_cache.repo_object.expiration_time = 300 +## max size of LRU, old values will be discarded if the size of cache reaches max_size +rc_cache.repo_object.max_size = 100 + + +################################ +### LOGGING CONFIGURATION #### +################################ +[loggers] +keys = root, vcsserver + +[handlers] +keys = console + +[formatters] +keys = generic + +############# +## LOGGERS ## +############# +[logger_root] +level = NOTSET +handlers = console + +[logger_vcsserver] +level = DEBUG +handlers = +qualname = vcsserver +propagate = 1 + + +############## +## HANDLERS ## +############## + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +################ +## FORMATTERS ## +################ + +[formatter_generic] +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/development_http.ini b/configs/development_http.ini deleted file mode 100644 --- a/configs/development_http.ini +++ /dev/null @@ -1,83 +0,0 @@ -################################################################################ -# RhodeCode VCSServer with HTTP Backend - configuration # -# # -################################################################################ - -[app:main] -use = egg:rhodecode-vcsserver - -pyramid.default_locale_name = en -pyramid.includes = - -# default locale used by VCS systems -locale = en_US.UTF-8 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - -# path to binaries for vcsserver, it should be set by the installer -# at installation time, e.g /home/user/vcsserver-1/profile/bin -core.binary_dir = "" - -[server:main] -## COMMON ## -host = 0.0.0.0 -port = 9900 - -use = egg:waitress#main - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/gunicorn_config.py b/configs/gunicorn_config.py new file mode 100644 --- /dev/null +++ b/configs/gunicorn_config.py @@ -0,0 +1,152 @@ +""" +gunicorn config extension and hooks. Sets additional configuration that is +available post the .ini config. + +- workers = ${cpu_number} +- threads = 1 +- proc_name = ${gunicorn_proc_name} +- worker_class = sync +- worker_connections = 10 +- max_requests = 1000 +- max_requests_jitter = 30 +- timeout = 21600 + +""" + +import multiprocessing +import sys +import time +import datetime +import threading +import traceback +from gunicorn.glogging import Logger + + +# GLOBAL +errorlog = '-' +accesslog = '-' +loglevel = 'debug' + +# SECURITY + +# The maximum size of HTTP request line in bytes. +limit_request_line = 4094 + +# Limit the number of HTTP headers fields in a request. +limit_request_fields = 1024 + +# Limit the allowed size of an HTTP request header field. +# Value is a positive number or 0. +# Setting it to 0 will allow unlimited header field sizes. +limit_request_field_size = 0 + + +# Timeout for graceful workers restart. +# After receiving a restart signal, workers have this much time to finish +# serving requests. Workers still alive after the timeout (starting from the +# receipt of the restart signal) are force killed. +graceful_timeout = 30 + + +# The number of seconds to wait for requests on a Keep-Alive connection. +# Generally set in the 1-5 seconds range. +keepalive = 2 + + +# SERVER MECHANICS +# None == system temp dir +# worker_tmp_dir is recommended to be set to some tmpfs +worker_tmp_dir = None +tmp_upload_dir = None + +# Custom log format +access_log_format = ( + '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') + +# self adjust workers based on CPU count +# workers = multiprocessing.cpu_count() * 2 + 1 + + +def post_fork(server, worker): + server.log.info("[<%-10s>] WORKER spawned", worker.pid) + + +def pre_fork(server, worker): + pass + + +def pre_exec(server): + server.log.info("Forked child, re-executing.") + + +def on_starting(server): + server.log.info("Server is starting.") + + +def when_ready(server): + server.log.info("Server is ready. Spawning workers") + + +def on_reload(server): + pass + + +def worker_int(worker): + worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid) + + # get traceback info, on worker crash + id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) + code = [] + for thread_id, stack in sys._current_frames().items(): + code.append( + "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id)) + for fname, lineno, name, line in traceback.extract_stack(stack): + code.append('File: "%s", line %d, in %s' % (fname, lineno, name)) + if line: + code.append(" %s" % (line.strip())) + worker.log.debug("\n".join(code)) + + +def worker_abort(worker): + worker.log.info("[<%-10s>] worker received SIGABRT signal", worker.pid) + + +def worker_exit(server, worker): + worker.log.info("[<%-10s>] worker exit", worker.pid) + + +def child_exit(server, worker): + worker.log.info("[<%-10s>] worker child exit", worker.pid) + + +def pre_request(worker, req): + worker.start_time = time.time() + worker.log.debug( + "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path) + + +def post_request(worker, req, environ, resp): + total_time = time.time() - worker.start_time + worker.log.debug( + "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.3fs", + worker.nr, req.method, req.path, resp.status_code, total_time) + + +class RhodeCodeLogger(Logger): + """ + Custom Logger that allows some customization that gunicorn doesn't allow + """ + + datefmt = r"%Y-%m-%d %H:%M:%S" + + def __init__(self, cfg): + Logger.__init__(self, cfg) + + def now(self): + """ return date in RhodeCode Log format """ + now = time.time() + msecs = int((now - long(now)) * 1000) + return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs) + + +logger_class = RhodeCodeLogger diff --git a/configs/production.ini b/configs/production.ini old mode 120000 new mode 100644 --- a/configs/production.ini +++ b/configs/production.ini @@ -1,1 +1,100 @@ -production_http.ini \ No newline at end of file +################################################################################ +# RhodeCode VCSServer with HTTP Backend - configuration # +# # +################################################################################ + + +[server:main] +## COMMON ## +host = 127.0.0.1 +port = 9900 + + +########################## +## GUNICORN WSGI SERVER ## +########################## +## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini +use = egg:gunicorn#main +## Sets the number of process workers. Recommended +## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers +workers = 2 +## process name +proc_name = rhodecode_vcsserver +## type of worker class, currently `sync` is the only option allowed. +worker_class = sync +## The maximum number of simultaneous clients. Valid only for Gevent +#worker_connections = 10 +## max number of requests that worker will handle before being gracefully +## restarted, could prevent memory leaks +max_requests = 1000 +max_requests_jitter = 30 +## amount of time a worker can spend with handling a request before it +## gets killed and restarted. Set to 6hrs +timeout = 21600 + + +[app:main] +use = egg:rhodecode-vcsserver + +pyramid.default_locale_name = en +pyramid.includes = + +## default locale used by VCS systems +locale = en_US.UTF-8 + + +## path to binaries for vcsserver, it should be set by the installer +## at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + +## cache region for storing repo_objects cache +rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru +## cache auto-expires after N seconds +rc_cache.repo_object.expiration_time = 300 +## max size of LRU, old values will be discarded if the size of cache reaches max_size +rc_cache.repo_object.max_size = 100 + + +################################ +### LOGGING CONFIGURATION #### +################################ +[loggers] +keys = root, vcsserver + +[handlers] +keys = console + +[formatters] +keys = generic + +############# +## LOGGERS ## +############# +[logger_root] +level = NOTSET +handlers = console + +[logger_vcsserver] +level = DEBUG +handlers = +qualname = vcsserver +propagate = 1 + + +############## +## HANDLERS ## +############## + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +################ +## FORMATTERS ## +################ + +[formatter_generic] +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S diff --git a/configs/production_http.ini b/configs/production_http.ini deleted file mode 100644 --- a/configs/production_http.ini +++ /dev/null @@ -1,106 +0,0 @@ -################################################################################ -# RhodeCode VCSServer with HTTP Backend - configuration # -# # -################################################################################ - - -[server:main] -## COMMON ## -host = 127.0.0.1 -port = 9900 - - -########################## -## GUNICORN WSGI SERVER ## -########################## -## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini -use = egg:gunicorn#main -## Sets the number of process workers. Recommended -## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers -workers = 2 -## process name -proc_name = rhodecode_vcsserver -## type of worker class, currently `sync` is the only option allowed. -worker_class = sync -## The maximum number of simultaneous clients. Valid only for Gevent -#worker_connections = 10 -## max number of requests that worker will handle before being gracefully -## restarted, could prevent memory leaks -max_requests = 1000 -max_requests_jitter = 30 -## amount of time a worker can spend with handling a request before it -## gets killed and restarted. Set to 6hrs -timeout = 21600 - - -[app:main] -use = egg:rhodecode-vcsserver - -pyramid.default_locale_name = en -pyramid.includes = - -# default locale used by VCS systems -locale = en_US.UTF-8 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - -# path to binaries for vcsserver, it should be set by the installer -# at installation time, e.g /home/user/vcsserver-1/profile/bin -core.binary_dir = "" - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -4,163 +4,175 @@ # derivation. For advanced tweaks to pimp up the development environment we use # "shell.nix" so that it does not have to clutter this file. -{ pkgs ? (import {}) -, pythonPackages ? "python27Packages" +args@ +{ pythonPackages ? "python27Packages" , pythonExternalOverrides ? self: super: {} -, doCheck ? true +, doCheck ? false +, ... }: -let pkgs_ = pkgs; in +let pkgs_ = (import {}); in let - pkgs = pkgs_.overridePackages (self: super: { - # bump GIT version - git = pkgs.lib.overrideDerivation pkgs_.git (oldAttrs: { - name = "git-2.16.4"; - src = pkgs.fetchurl { - url = "https://www.kernel.org/pub/software/scm/git/git-2.16.4.tar.xz"; - sha256 = "0cnmidjvbdf81mybcvxvl0c2r2x2nvq2jj2dl59dmrc7qklv0sbf"; - }; - - patches = [ - ./pkgs/git_patches/docbook2texi.patch - ./pkgs/git_patches/symlinks-in-bin.patch - ./pkgs/git_patches/git-sh-i18n.patch - ./pkgs/git_patches/ssh-path.patch - ]; - - }); - # Override subversion derivation to - # - activate python bindings - subversion = let - subversionWithPython = super.subversion.override { - httpSupport = true; - pythonBindings = true; - python = self.python27Packages.python; - }; - - in + # TODO: Currently we ignore the passed in pkgs, instead we should use it + # somehow as a base and apply overlays to it. + pkgs = import { + overlays = [ + (import ./pkgs/overlays.nix) + ]; + inherit (pkgs_) + system; + }; - pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: { - name = "subversion-1.9.7"; - src = pkgs.fetchurl { - url = "https://www.apache.org/dist/subversion/subversion-1.9.7.tar.gz"; - sha256 = "0g3cs2h008z8ymgkhbk54jp87bjh7y049rn42igj881yi2f20an7"; - }; - - }); - - }); + # Works with the new python-packages, still can fallback to the old + # variant. + basePythonPackagesUnfix = basePythonPackages.__unfix__ or ( + self: basePythonPackages.override (a: { inherit self; })); - inherit (pkgs.lib) fix extends; - basePythonPackages = with builtins; if isAttrs pythonPackages - then pythonPackages - else getAttr pythonPackages pkgs; + # Evaluates to the last segment of a file system path. + basename = path: with pkgs.lib; last (splitString "/" path); - elem = builtins.elem; - basename = path: with pkgs.lib; last (splitString "/" path); - startsWith = prefix: full: let - actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full; - in actualPrefix == prefix; - + # source code filter used as arugment to builtins.filterSource. src-filter = path: type: with pkgs.lib; let ext = last (splitString "." path); in - !elem (basename path) [".hg" ".git" "__pycache__" ".eggs" - "node_modules" "build" "data" "tmp"] && - !elem ext ["egg-info" "pyc"] && - !startsWith "result" path; + !builtins.elem (basename path) [ + ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev" + "bower_components" "node_modules" + "build" "data" "result" "tmp"] && + !builtins.elem ext ["egg-info" "pyc"] && + # TODO: johbo: This check is wrong, since "path" contains an absolute path, + # it would still be good to restore it since we want to ignore "result-*". + !hasPrefix "result" path; + sources = + let + inherit (pkgs.lib) all isString attrValues; + sourcesConfig = pkgs.config.rc.sources or {}; + in + # Ensure that sources are configured as strings. Using a path + # would result in a copy into the nix store. + assert all isString (attrValues sourcesConfig); + sourcesConfig; + + version = builtins.readFile "${rhodecode-vcsserver-src}/vcsserver/VERSION"; rhodecode-vcsserver-src = builtins.filterSource src-filter ./.; - pythonGeneratedPackages = self: basePythonPackages.override (a: { - inherit self; - }) // (scopedImport { - self = self; - super = basePythonPackages; - inherit pkgs; - inherit (pkgs) fetchurl fetchgit; - } ./pkgs/python-packages.nix); - - pythonOverrides = import ./pkgs/python-packages-overrides.nix { - inherit basePythonPackages pkgs; - }; - - version = builtins.readFile ./vcsserver/VERSION; - pythonLocalOverrides = self: super: { - rhodecode-vcsserver = super.rhodecode-vcsserver.override (attrs: { - inherit doCheck version; + rhodecode-vcsserver = + let + releaseName = "RhodeCodeVCSServer-${version}"; + in super.rhodecode-vcsserver.override (attrs: { + inherit + doCheck + version; name = "rhodecode-vcsserver-${version}"; - releaseName = "RhodeCodeVCSServer-${version}"; + releaseName = releaseName; src = rhodecode-vcsserver-src; dontStrip = true; # prevent strip, we don't need it. - propagatedBuildInputs = attrs.propagatedBuildInputs ++ ([ - pkgs.git - pkgs.subversion - ]); - - # TODO: johbo: Make a nicer way to expose the parts. Maybe - # pkgs/default.nix? + # expose following attributed outside passthru = { pythonPackages = self; }; - # Add VCSServer bin directory to path so that tests can find 'vcsserver'. + propagatedBuildInputs = + attrs.propagatedBuildInputs or [] ++ [ + pkgs.git + pkgs.subversion + ]; + + # set some default locale env variables + LC_ALL = "en_US.UTF-8"; + LOCALE_ARCHIVE = + if pkgs.stdenv.isLinux + then "${pkgs.glibcLocales}/lib/locale/locale-archive" + else ""; + + # Add bin directory to path so that tests can find 'vcsserver'. preCheck = '' export PATH="$out/bin:$PATH" ''; - # put custom attrs here + # custom check phase for testing checkPhase = '' runHook preCheck - PYTHONHASHSEED=random py.test -p no:sugar -vv --cov-config=.coveragerc --cov=vcsserver --cov-report=term-missing vcsserver + PYTHONHASHSEED=random py.test -vv -p no:sugar -r xw --cov-config=.coveragerc --cov=vcsserver --cov-report=term-missing vcsserver runHook postCheck ''; + postCheck = '' + echo "Cleanup of vcsserver/tests" + rm -rf $out/lib/${self.python.libPrefix}/site-packages/vcsserver/tests + ''; + postInstall = '' - echo "Writing meta information for rccontrol to nix-support/rccontrol" + echo "Writing vcsserver meta information for rccontrol to nix-support/rccontrol" mkdir -p $out/nix-support/rccontrol cp -v vcsserver/VERSION $out/nix-support/rccontrol/version - echo "DONE: Meta information for rccontrol written" + echo "DONE: vcsserver meta information for rccontrol written" + + mkdir -p $out/etc + cp configs/production.ini $out/etc + echo "DONE: saved vcsserver production.ini into $out/etc" # python based programs need to be wrapped + mkdir -p $out/bin + ln -s ${self.python}/bin/python $out/bin ln -s ${self.pyramid}/bin/* $out/bin/ ln -s ${self.gunicorn}/bin/gunicorn $out/bin/ # Symlink version control utilities - # # We ensure that always the correct version is available as a symlink. # So that users calling them via the profile path will always use the # correct version. - ln -s ${self.python}/bin/python $out/bin + ln -s ${pkgs.git}/bin/git $out/bin ln -s ${self.mercurial}/bin/hg $out/bin ln -s ${pkgs.subversion}/bin/svn* $out/bin + echo "DONE: created symlinks into $out/bin" for file in $out/bin/*; do wrapProgram $file \ - --set PATH $PATH \ - --set PYTHONPATH $PYTHONPATH \ + --prefix PATH : $PATH \ + --prefix PYTHONPATH : $PYTHONPATH \ --set PYTHONHASHSEED random done + echo "DONE: vcsserver binary wrapping" ''; }); }; + basePythonPackages = with builtins; + if isAttrs pythonPackages then + pythonPackages + else + getAttr pythonPackages pkgs; + + pythonGeneratedPackages = import ./pkgs/python-packages.nix { + inherit pkgs; + inherit (pkgs) fetchurl fetchgit fetchhg; + }; + + pythonVCSServerOverrides = import ./pkgs/python-packages-overrides.nix { + inherit pkgs basePythonPackages; + }; + + # Apply all overrides and fix the final package set - myPythonPackages = - (fix + myPythonPackagesUnfix = with pkgs.lib; (extends pythonExternalOverrides (extends pythonLocalOverrides - (extends pythonOverrides - pythonGeneratedPackages)))); + (extends pythonVCSServerOverrides + (extends pythonGeneratedPackages + basePythonPackagesUnfix)))); + + myPythonPackages = (pkgs.lib.fix myPythonPackagesUnfix); in myPythonPackages.rhodecode-vcsserver diff --git a/pkgs/README.rst b/pkgs/README.rst new file mode 100644 --- /dev/null +++ b/pkgs/README.rst @@ -0,0 +1,28 @@ + +============================== + Generate the Nix expressions +============================== + +Details can be found in the repository of `RhodeCode Enterprise CE`_ inside of +the file `docs/contributing/dependencies.rst`. + +Start the environment as follows: + +.. code:: shell + + nix-shell pkgs/shell-generate.nix + + +Python dependencies +=================== + +.. code:: shell + + pip2nix generate --licenses + # or faster + nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses" + + +.. Links + +.. _RhodeCode Enterprise CE: https://code.rhodecode.com/rhodecode-enterprise-ce diff --git a/pkgs/nix-common/pip2nix.nix b/pkgs/nix-common/pip2nix.nix new file mode 100755 --- /dev/null +++ b/pkgs/nix-common/pip2nix.nix @@ -0,0 +1,17 @@ +{ pkgs +, pythonPackages +}: + +rec { + pip2nix-src = pkgs.fetchzip { + url = https://github.com/johbo/pip2nix/archive/51e6fdae34d0e8ded9efeef7a8601730249687a6.tar.gz; + sha256 = "02a4jjgi7lsvf8mhrxsd56s9a3yg20081rl9bgc2m84w60v2gbz2"; + }; + + pip2nix = import pip2nix-src { + inherit + pkgs + pythonPackages; + }; + +} diff --git a/pkgs/overlays.nix b/pkgs/overlays.nix new file mode 100755 --- /dev/null +++ b/pkgs/overlays.nix @@ -0,0 +1,45 @@ +self: super: { + # bump GIT version + git = super.lib.overrideDerivation super.git (oldAttrs: { + name = "git-2.17.1"; + src = self.fetchurl { + url = "https://www.kernel.org/pub/software/scm/git/git-2.17.1.tar.xz"; + sha256 = "0pm6bdnrrm165k3krnazxcxadifk2gqi30awlbcf9fism1x6w4vr"; + }; + + patches = [ + ./git_patches/docbook2texi.patch + ./git_patches/symlinks-in-bin.patch + ./git_patches/git-sh-i18n.patch + ./git_patches/ssh-path.patch + ]; + + }); + + # Override subversion derivation to + # - activate python bindings + subversion = + let + subversionWithPython = super.subversion.override { + httpSupport = true; + pythonBindings = true; + python = self.python27Packages.python; + }; + in + super.lib.overrideDerivation subversionWithPython (oldAttrs: { + name = "subversion-1.10.2"; + src = self.fetchurl { + url = "https://archive.apache.org/dist/subversion/subversion-1.10.2.tar.gz"; + sha256 = "0xv5z2bg0lw7057g913yc13f60nfj257wvmsq22pr33m4syf26sg"; + }; + + ## use internal lz4/utf8proc because it is stable and shipped with SVN + configureFlags = oldAttrs.configureFlags ++ [ + " --with-lz4=internal" + " --with-utf8proc=internal" + ]; + + + }); + +} diff --git a/pkgs/patch-beaker-lock-func-debug.diff b/pkgs/patch-beaker-lock-func-debug.diff deleted file mode 100644 --- a/pkgs/patch-beaker-lock-func-debug.diff +++ /dev/null @@ -1,20 +0,0 @@ -diff -rup Beaker-1.9.1-orig/beaker/container.py Beaker-1.9.1/beaker/container.py ---- Beaker-1.9.1-orig/beaker/container.py 2018-04-10 10:23:04.000000000 +0200 -+++ Beaker-1.9.1/beaker/container.py 2018-04-10 10:23:34.000000000 +0200 -@@ -353,13 +353,13 @@ class Value(object): - debug("get_value returning old value while new one is created") - return value - else: -- debug("lock_creatfunc (didnt wait)") -+ debug("lock_creatfunc `%s` (didnt wait)", self.createfunc.__name__) - has_createlock = True - - if not has_createlock: -- debug("lock_createfunc (waiting)") -+ debug("lock_createfunc `%s` (waiting)", self.createfunc.__name__) - creation_lock.acquire() -- debug("lock_createfunc (waited)") -+ debug("lock_createfunc `%s` (waited)", self.createfunc.__name__) - - try: - # see if someone created the value already diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -4,57 +4,50 @@ # python-packages.nix. The main objective is to add needed dependencies of C # libraries and tweak the build instructions where needed. -{ pkgs, basePythonPackages }: +{ pkgs +, basePythonPackages +}: let sed = "sed -i"; + in self: super: { - Beaker = super.Beaker.override (attrs: { - patches = [ - ./patch-beaker-lock-func-debug.diff + "gevent" = super."gevent".override (attrs: { + propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ + # NOTE: (marcink) odd requirements from gevent aren't set properly, + # thus we need to inject psutil manually + self."psutil" ]; }); - subvertpy = super.subvertpy.override (attrs: { - # TODO: johbo: Remove the "or" once we drop 16.03 support - SVN_PREFIX = "${pkgs.subversion.dev or pkgs.subversion}"; + "hgsubversion" = super."hgsubversion".override (attrs: { propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ + pkgs.sqlite + #basePythonPackages.sqlite3 + self.mercurial + ]; + }); + + "subvertpy" = super."subvertpy".override (attrs: { + SVN_PREFIX = "${pkgs.subversion.dev}"; + propagatedBuildInputs = [ + pkgs.apr.dev pkgs.aprutil pkgs.subversion ]; - preBuild = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' - ${sed} -e "s/'gcc'/'clang'/" setup.py - ''; }); - hgsubversion = super.hgsubversion.override (attrs: { - propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ - pkgs.sqlite - basePythonPackages.sqlite3 + "mercurial" = super."mercurial".override (attrs: { + propagatedBuildInputs = [ + # self.python.modules.curses ]; }); - mercurial = super.mercurial.override (attrs: { - propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ - self.python.modules.curses - ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin - pkgs.darwin.apple_sdk.frameworks.ApplicationServices; - }); - - pyramid = super.pyramid.override (attrs: { - postFixup = '' - wrapPythonPrograms - # TODO: johbo: "wrapPython" adds this magic line which - # confuses pserve. - ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped - ''; - }); - - # Avoid that setuptools is replaced, this leads to trouble - # with buildPythonPackage. - setuptools = basePythonPackages.setuptools; + # Avoid that base packages screw up the build process + inherit (basePythonPackages) + setuptools; } diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1,873 +1,943 @@ -# Generated by pip2nix 0.4.0 +# Generated by pip2nix 0.8.0.dev1 # See https://github.com/johbo/pip2nix -{ - Beaker = super.buildPythonPackage { - name = "Beaker-1.9.1"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [funcsigs]; - src = fetchurl { - url = "https://pypi.python.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz"; - md5 = "46fda0a164e2b0d24ccbda51a2310301"; - }; - meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; - }; - }; - Jinja2 = super.buildPythonPackage { - name = "Jinja2-2.9.6"; - buildInputs = with self; []; +{ pkgs, fetchurl, fetchgit, fetchhg }: + +self: super: { + "atomicwrites" = super.buildPythonPackage { + name = "atomicwrites-1.1.5"; doCheck = false; - propagatedBuildInputs = with self; [MarkupSafe]; src = fetchurl { - url = "https://pypi.python.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz"; - md5 = "6411537324b4dba0956aaa8109f3c77b"; - }; - meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; - }; - }; - Mako = super.buildPythonPackage { - name = "Mako-1.0.7"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [MarkupSafe]; - src = fetchurl { - url = "https://pypi.python.org/packages/eb/f3/67579bb486517c0d49547f9697e36582cd19dafb5df9e687ed8e22de57fa/Mako-1.0.7.tar.gz"; - md5 = "5836cc997b1b773ef389bf6629c30e65"; + url = "https://files.pythonhosted.org/packages/a1/e1/2d9bc76838e6e6667fde5814aa25d7feb93d6fa471bf6816daac2596e8b2/atomicwrites-1.1.5.tar.gz"; + sha256 = "11bm90fwm2avvf4f3ib8g925w7jr4m11vcsinn1bi6ns4bm32214"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - MarkupSafe = super.buildPythonPackage { - name = "MarkupSafe-1.0"; - buildInputs = with self; []; + "attrs" = super.buildPythonPackage { + name = "attrs-18.1.0"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz"; - md5 = "2fcedc9284d50e577b5192e8e3578355"; - }; - meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; - }; - }; - PasteDeploy = super.buildPythonPackage { - name = "PasteDeploy-1.5.2"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; []; - src = fetchurl { - url = "https://pypi.python.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz"; - md5 = "352b7205c78c8de4987578d19431af3b"; + url = "https://files.pythonhosted.org/packages/e4/ac/a04671e118b57bee87dabca1e0f2d3bda816b7a551036012d0ca24190e71/attrs-18.1.0.tar.gz"; + sha256 = "0yzqz8wv3w1srav5683a55v49i0szkm47dyrnkd56fqs8j8ypl70"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - WebOb = super.buildPythonPackage { - name = "WebOb-1.7.4"; - buildInputs = with self; []; + "backports.shutil-get-terminal-size" = super.buildPythonPackage { + name = "backports.shutil-get-terminal-size-1.0.0"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/75/34/731e23f52371852dfe7490a61644826ba7fe70fd52a377aaca0f4956ba7f/WebOb-1.7.4.tar.gz"; - md5 = "397e46892d7f199b1a07eb20a2d3d9bd"; + url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz"; + sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - WebTest = super.buildPythonPackage { - name = "WebTest-2.0.29"; - buildInputs = with self; []; + "beautifulsoup4" = super.buildPythonPackage { + name = "beautifulsoup4-4.6.3"; doCheck = false; - propagatedBuildInputs = with self; [six WebOb waitress beautifulsoup4]; src = fetchurl { - url = "https://pypi.python.org/packages/94/de/8f94738be649997da99c47b104aa3c3984ecec51a1d8153ed09638253d56/WebTest-2.0.29.tar.gz"; - md5 = "30b4cf0d340b9a5335fac4389e6f84fc"; + url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz"; + sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - backports.shutil-get-terminal-size = super.buildPythonPackage { - name = "backports.shutil-get-terminal-size-1.0.0"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; []; - src = fetchurl { - url = "https://pypi.python.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz"; - md5 = "03267762480bd86b50580dc19dff3c66"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; - beautifulsoup4 = super.buildPythonPackage { - name = "beautifulsoup4-4.6.0"; - buildInputs = with self; []; + "configobj" = super.buildPythonPackage { + name = "configobj-5.0.6"; doCheck = false; - propagatedBuildInputs = with self; []; + propagatedBuildInputs = [ + self."six" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/fa/8d/1d14391fdaed5abada4e0f63543fef49b8331a34ca60c88bd521bcf7f782/beautifulsoup4-4.6.0.tar.gz"; - md5 = "c17714d0f91a23b708a592cb3c697728"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; - configobj = super.buildPythonPackage { - name = "configobj-5.0.6"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [six]; - src = fetchurl { - url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz"; - md5 = "e472a3a1c2a67bb0ec9b5d54c13a47d6"; + url = "https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c"; + sha256 = "1hhcxirwvg58grlfr177b3awhbq8hlx1l3lh69ifl1ki7lfd1s1x"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - cov-core = super.buildPythonPackage { + "cov-core" = super.buildPythonPackage { name = "cov-core-1.15.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; [coverage]; + propagatedBuildInputs = [ + self."coverage" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz"; - md5 = "f519d4cb4c4e52856afb14af52919fe6"; + url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz"; + sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - coverage = super.buildPythonPackage { + "coverage" = super.buildPythonPackage { name = "coverage-3.7.1"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz"; - md5 = "c47b36ceb17eaff3ecfab3bcd347d0df"; + url = "https://files.pythonhosted.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz"; + sha256 = "0knlbq79g2ww6xzsyknj9rirrgrgc983dpa2d9nkdf31mb2a3bni"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - decorator = super.buildPythonPackage { + "decorator" = super.buildPythonPackage { name = "decorator-4.1.2"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz"; - md5 = "a0f7f4fe00ae2dde93494d90c192cf8c"; + url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz"; + sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ]; }; }; - dulwich = super.buildPythonPackage { - name = "dulwich-0.13.0"; - buildInputs = with self; []; + "dogpile.cache" = super.buildPythonPackage { + name = "dogpile.cache-0.6.6"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/48/ca/604154d835c3668efb8a31bd979b0ea4bf39c2934a40ffecc0662296cb51/dogpile.cache-0.6.6.tar.gz"; + sha256 = "1h8n1lxd4l2qvahfkiinljkqz7pww7w3sgag0j8j9ixbl2h4wk84"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + "dogpile.core" = super.buildPythonPackage { + name = "dogpile.core-0.4.1"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz"; - md5 = "6dede0626657c2bd08f48ca1221eea91"; + url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz"; + sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + "dulwich" = super.buildPythonPackage { + name = "dulwich-0.13.0"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz"; + sha256 = "0f1jwvrh549c4rgavkn3wizrch904s73s4fmrxykxy9cw8s57lwf"; }; meta = { license = [ pkgs.lib.licenses.gpl2Plus ]; }; }; - enum34 = super.buildPythonPackage { + "enum34" = super.buildPythonPackage { name = "enum34-1.1.6"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; - md5 = "5f13a0841a61f7fc295c514490d120d0"; + url = "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz"; + sha256 = "1cgm5ng2gcfrkrm3hc22brl6chdmv67b9zvva9sfs7gn7dwc9n4a"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - funcsigs = super.buildPythonPackage { + "funcsigs" = super.buildPythonPackage { name = "funcsigs-1.0.2"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz"; - md5 = "7e583285b1fb8a76305d6d68f4ccc14e"; + url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz"; + sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7"; }; meta = { license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ]; }; }; - gevent = super.buildPythonPackage { - name = "gevent-1.2.2"; - buildInputs = with self; []; + "gevent" = super.buildPythonPackage { + name = "gevent-1.3.5"; doCheck = false; - propagatedBuildInputs = with self; [greenlet]; + propagatedBuildInputs = [ + self."greenlet" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/1b/92/b111f76e54d2be11375b47b213b56687214f258fd9dae703546d30b837be/gevent-1.2.2.tar.gz"; - md5 = "7f0baf355384fe5ff2ecf66853422554"; + url = "https://files.pythonhosted.org/packages/e6/0a/fc345c6e6161f84484870dbcaa58e427c10bd9bdcd08a69bed3d6b398bf1/gevent-1.3.5.tar.gz"; + sha256 = "1w3gydxirgd2f60c5yv579w4903ds9s4g3587ik4jby97hgqc5bz"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - gprof2dot = super.buildPythonPackage { + "gprof2dot" = super.buildPythonPackage { name = "gprof2dot-2017.9.19"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz"; - md5 = "cda2d552bb0d0b9f16e6824a9aabd225"; + url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz"; + sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f"; }; meta = { license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ]; }; }; - greenlet = super.buildPythonPackage { + "greenlet" = super.buildPythonPackage { name = "greenlet-0.4.13"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz"; - md5 = "6e0b9dd5385f81d478451ec8ed1d62b3"; + url = "https://files.pythonhosted.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz"; + sha256 = "1r412gfx25jrdiv444prmz5a8igrfabwnwqyr6b52ypq7ga87vqg"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "gunicorn" = super.buildPythonPackage { + name = "gunicorn-19.9.0"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz"; + sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - gunicorn = super.buildPythonPackage { - name = "gunicorn-19.7.1"; - buildInputs = with self; []; + "hg-evolve" = super.buildPythonPackage { + name = "hg-evolve-8.0.1"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/06/1a/c5c12d8f117426f05285a820ee5a23121882f5381104e86276b72598934f/hg-evolve-8.0.1.tar.gz"; + sha256 = "1brafifb42k71gl7qssb5m3ijnm7y30lfvm90z8xxcr2fgz19p29"; + }; + meta = { + license = [ { fullName = "GPLv2+"; } ]; + }; + }; + "hgsubversion" = super.buildPythonPackage { + name = "hgsubversion-1.9.2"; doCheck = false; - propagatedBuildInputs = with self; []; + propagatedBuildInputs = [ + self."mercurial" + self."subvertpy" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/30/3a/10bb213cede0cc4d13ac2263316c872a64bf4c819000c8ccd801f1d5f822/gunicorn-19.7.1.tar.gz"; - md5 = "174d3c3cd670a5be0404d84c484e590c"; + url = "https://files.pythonhosted.org/packages/05/80/3a3cef10dd65e86528ef8d7ac57a41ebc782d0f3c6cfa4fed021aa9fbee0/hgsubversion-1.9.2.tar.gz"; + sha256 = "16490narhq14vskml3dam8g5y3w3hdqj3g8bgm2b0c0i85l1xvcz"; + }; + meta = { + license = [ pkgs.lib.licenses.gpl1 ]; + }; + }; + "hupper" = super.buildPythonPackage { + name = "hupper-1.3"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/51/0c/96335b1f2f32245fb871eea5bb9773196505ddb71fad15190056a282df9e/hupper-1.3.tar.gz"; + sha256 = "1pkyrm9c2crc32ps00k1ahnc5clj3pjwiarc7j0x8aykwih7ff10"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - hg-evolve = super.buildPythonPackage { - name = "hg-evolve-7.0.1"; - buildInputs = with self; []; + "ipdb" = super.buildPythonPackage { + name = "ipdb-0.11"; doCheck = false; - propagatedBuildInputs = with self; []; + propagatedBuildInputs = [ + self."setuptools" + self."ipython" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/92/5c/4c216be1a08f326a12076b645f4892a2b0865810db1f4a0c9648f1f4c113/hg-evolve-7.0.1.tar.gz"; - md5 = "2dfa926846ea873a8406bababb06b277"; + url = "https://files.pythonhosted.org/packages/80/fe/4564de08f174f3846364b3add8426d14cebee228f741c27e702b2877e85b/ipdb-0.11.tar.gz"; + sha256 = "02m0l8wrhhd3z7dg3czn5ys1g5pxib516hpshdzp7rxzsxgcd0bh"; }; meta = { - license = [ { fullName = "GPLv2+"; } ]; + license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - hgsubversion = super.buildPythonPackage { - name = "hgsubversion-1.9"; - buildInputs = with self; []; + "ipython" = super.buildPythonPackage { + name = "ipython-5.1.0"; doCheck = false; - propagatedBuildInputs = with self; [mercurial subvertpy]; + propagatedBuildInputs = [ + self."setuptools" + self."decorator" + self."pickleshare" + self."simplegeneric" + self."traitlets" + self."prompt-toolkit" + self."pygments" + self."pexpect" + self."backports.shutil-get-terminal-size" + self."pathlib2" + self."pexpect" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/db/26/7293a6c6b85e2a74ab452e9ba7f00b04ff0e440e6cd4f84131ac5d5e6b22/hgsubversion-1.9.tar.gz"; - md5 = "0c6f93ef12cc2e7fe67286f16bcc7211"; + url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz"; + sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y"; }; meta = { - license = [ pkgs.lib.licenses.gpl1 ]; + license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - hupper = super.buildPythonPackage { - name = "hupper-1.0"; - buildInputs = with self; []; + "ipython-genutils" = super.buildPythonPackage { + name = "ipython-genutils-0.2.0"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/2e/07/df892c564dc09bb3cf6f6deb976c26adf9117db75ba218cb4353dbc9d826/hupper-1.0.tar.gz"; - md5 = "26e77da7d5ac5858f59af050d1a6eb5a"; + url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz"; + sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + "mako" = super.buildPythonPackage { + name = "mako-1.0.7"; + doCheck = false; + propagatedBuildInputs = [ + self."markupsafe" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/eb/f3/67579bb486517c0d49547f9697e36582cd19dafb5df9e687ed8e22de57fa/Mako-1.0.7.tar.gz"; + sha256 = "1bi5gnr8r8dva06qpyx4kgjc6spm2k1y908183nbbaylggjzs0jf"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - infrae.cache = super.buildPythonPackage { - name = "infrae.cache-1.0.1"; - buildInputs = with self; []; + "markupsafe" = super.buildPythonPackage { + name = "markupsafe-1.0"; doCheck = false; - propagatedBuildInputs = with self; [Beaker repoze.lru]; src = fetchurl { - url = "https://pypi.python.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz"; - md5 = "b09076a766747e6ed2a755cc62088e32"; - }; - meta = { - license = [ pkgs.lib.licenses.zpt21 ]; - }; - }; - ipdb = super.buildPythonPackage { - name = "ipdb-0.10.3"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [setuptools ipython]; - src = fetchurl { - url = "https://pypi.python.org/packages/ad/cc/0e7298e1fbf2efd52667c9354a12aa69fb6f796ce230cca03525051718ef/ipdb-0.10.3.tar.gz"; - md5 = "def1f6ac075d54bdee07e6501263d4fa"; + url = "https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz"; + sha256 = "0rdn1s8x9ni7ss8rfiacj7x1085lx8mh2zdwqslnw8xc3l4nkgm6"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - ipython = super.buildPythonPackage { - name = "ipython-5.1.0"; - buildInputs = with self; []; + "mercurial" = super.buildPythonPackage { + name = "mercurial-4.6.2"; doCheck = false; - propagatedBuildInputs = with self; [setuptools decorator pickleshare simplegeneric traitlets prompt-toolkit pygments pexpect backports.shutil-get-terminal-size pathlib2 pexpect]; src = fetchurl { - url = "https://pypi.python.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz"; - md5 = "47c8122420f65b58784cb4b9b4af35e3"; + url = "https://files.pythonhosted.org/packages/d9/fb/c7ecf2b7fd349878dbf45b8390b8db735cef73d49dd9ce8a364b4ca3a846/mercurial-4.6.2.tar.gz"; + sha256 = "1bv6wgcdx8glihjjfg22khhc52mclsn4kwfqvzbzlg0b42h4xl0w"; }; meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; + license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ]; }; }; - ipython-genutils = super.buildPythonPackage { - name = "ipython-genutils-0.2.0"; - buildInputs = with self; []; + "mock" = super.buildPythonPackage { + name = "mock-1.0.1"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz"; - md5 = "5a4f9781f78466da0ea1a648f3e1f79f"; + url = "https://files.pythonhosted.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz"; + sha256 = "0kzlsbki6q0awf89rc287f3aj8x431lrajf160a70z0ikhnxsfdq"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - mercurial = super.buildPythonPackage { - name = "mercurial-4.4.2"; - buildInputs = with self; []; + "more-itertools" = super.buildPythonPackage { + name = "more-itertools-4.3.0"; doCheck = false; - propagatedBuildInputs = with self; []; + propagatedBuildInputs = [ + self."six" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/d0/83/92a5fa662ba277128db305e39e7ea5a638f2f1cbbc6dc5fbf4c14aefae22/mercurial-4.4.2.tar.gz"; - md5 = "95769125cf7e9dbc341a983253acefcd"; + url = "https://files.pythonhosted.org/packages/88/ff/6d485d7362f39880810278bdc906c13300db05485d9c65971dec1142da6a/more-itertools-4.3.0.tar.gz"; + sha256 = "17h3na0rdh8xq30w4b9pizgkdxmm51896bxw600x84jflg9vaxn4"; }; meta = { - license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ]; + license = [ pkgs.lib.licenses.mit ]; }; }; - mock = super.buildPythonPackage { - name = "mock-1.0.1"; - buildInputs = with self; []; + "msgpack-python" = super.buildPythonPackage { + name = "msgpack-python-0.5.6"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz"; - md5 = "c3971991738caa55ec7c356bbc154ee2"; - }; - meta = { - license = [ pkgs.lib.licenses.bsdOriginal ]; - }; - }; - msgpack-python = super.buildPythonPackage { - name = "msgpack-python-0.4.8"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; []; - src = fetchurl { - url = "https://pypi.python.org/packages/21/27/8a1d82041c7a2a51fcc73675875a5f9ea06c2663e02fcfeb708be1d081a0/msgpack-python-0.4.8.tar.gz"; - md5 = "dcd854fb41ee7584ebbf35e049e6be98"; + url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz"; + sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p"; }; meta = { license = [ pkgs.lib.licenses.asl20 ]; }; }; - pathlib2 = super.buildPythonPackage { - name = "pathlib2-2.3.0"; - buildInputs = with self; []; + "pastedeploy" = super.buildPythonPackage { + name = "pastedeploy-1.5.2"; doCheck = false; - propagatedBuildInputs = with self; [six scandir]; src = fetchurl { - url = "https://pypi.python.org/packages/a1/14/df0deb867c2733f7d857523c10942b3d6612a1b222502fdffa9439943dfb/pathlib2-2.3.0.tar.gz"; - md5 = "89c90409d11fd5947966b6a30a47d18c"; + url = "https://files.pythonhosted.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz"; + sha256 = "1jz3m4hq8v6hyhfjz9425nd3nvn52cvbfipdcd72krjmla4qz1fm"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "pathlib2" = super.buildPythonPackage { + name = "pathlib2-2.3.0"; + doCheck = false; + propagatedBuildInputs = [ + self."six" + self."scandir" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/a1/14/df0deb867c2733f7d857523c10942b3d6612a1b222502fdffa9439943dfb/pathlib2-2.3.0.tar.gz"; + sha256 = "1cx5gs2v9j2vnzmcrbq5l8fq2mwrr1h6pyf1sjdji2w1bavm09fk"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - pexpect = super.buildPythonPackage { - name = "pexpect-4.4.0"; - buildInputs = with self; []; + "pexpect" = super.buildPythonPackage { + name = "pexpect-4.6.0"; doCheck = false; - propagatedBuildInputs = with self; [ptyprocess]; + propagatedBuildInputs = [ + self."ptyprocess" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/fa/c3/60c0cbf96f242d0b47a82e9ca634dcd6dcb043832cf05e17540812e1c707/pexpect-4.4.0.tar.gz"; - md5 = "e9b07f0765df8245ac72201d757baaef"; + url = "https://files.pythonhosted.org/packages/89/43/07d07654ee3e25235d8cea4164cdee0ec39d1fda8e9203156ebe403ffda4/pexpect-4.6.0.tar.gz"; + sha256 = "1fla85g47iaxxpjhp9vkxdnv4pgc7rplfy6ja491smrrk0jqi3ia"; }; meta = { license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ]; }; }; - pickleshare = super.buildPythonPackage { + "pickleshare" = super.buildPythonPackage { name = "pickleshare-0.7.4"; - buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = [ + self."pathlib2" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz"; + sha256 = "0yvk14dzxk7g6qpr7iw23vzqbsr0dh4ij4xynkhnzpfz4xr2bac4"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "plaster" = super.buildPythonPackage { + name = "plaster-1.0"; doCheck = false; - propagatedBuildInputs = with self; [pathlib2]; + propagatedBuildInputs = [ + self."setuptools" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz"; - md5 = "6a9e5dd8dfc023031f6b7b3f824cab12"; + url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz"; + sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "plaster-pastedeploy" = super.buildPythonPackage { + name = "plaster-pastedeploy-0.6"; + doCheck = false; + propagatedBuildInputs = [ + self."pastedeploy" + self."plaster" + ]; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/3f/e7/6a6833158d2038ec40085433308a1e164fd1dac595513f6dd556d5669bb8/plaster_pastedeploy-0.6.tar.gz"; + sha256 = "1bkggk18f4z2bmsmxyxabvf62znvjwbivzh880419r3ap0616cf2"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "pluggy" = super.buildPythonPackage { + name = "pluggy-0.6.0"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/11/bf/cbeb8cdfaffa9f2ea154a30ae31a9d04a1209312e2919138b4171a1f8199/pluggy-0.6.0.tar.gz"; + sha256 = "1zqckndfn85l1cd8pndw212zg1bq9fkg1nnj32kp2mppppsyg2kz"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - plaster = super.buildPythonPackage { - name = "plaster-1.0"; - buildInputs = with self; []; + "prompt-toolkit" = super.buildPythonPackage { + name = "prompt-toolkit-1.0.15"; doCheck = false; - propagatedBuildInputs = with self; [setuptools]; + propagatedBuildInputs = [ + self."six" + self."wcwidth" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz"; - md5 = "80e6beb4760c16fea31754babcc0576e"; + url = "https://files.pythonhosted.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz"; + sha256 = "05v9h5nydljwpj5nm8n804ms0glajwfy1zagrzqrg91wk3qqi1c5"; }; meta = { - license = [ pkgs.lib.licenses.mit ]; + license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - plaster-pastedeploy = super.buildPythonPackage { - name = "plaster-pastedeploy-0.4.2"; - buildInputs = with self; []; + "psutil" = super.buildPythonPackage { + name = "psutil-5.4.6"; doCheck = false; - propagatedBuildInputs = with self; [PasteDeploy plaster]; src = fetchurl { - url = "https://pypi.python.org/packages/2c/62/0daf9c0be958e785023e583e51baac15863699e956bfb3d448898d80edd8/plaster_pastedeploy-0.4.2.tar.gz"; - md5 = "58fd7852002909378e818c9d5b71e90a"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; - prompt-toolkit = super.buildPythonPackage { - name = "prompt-toolkit-1.0.15"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [six wcwidth]; - src = fetchurl { - url = "https://pypi.python.org/packages/8a/ad/cf6b128866e78ad6d7f1dc5b7f99885fb813393d9860778b2984582e81b5/prompt_toolkit-1.0.15.tar.gz"; - md5 = "8fe70295006dbc8afedd43e5eba99032"; + url = "https://files.pythonhosted.org/packages/51/9e/0f8f5423ce28c9109807024f7bdde776ed0b1161de20b408875de7e030c3/psutil-5.4.6.tar.gz"; + sha256 = "1xmw4qi6hnrhw81xqzkvmsm9im7j2vkk4v26ycjwq2jczqsmlvk8"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - ptyprocess = super.buildPythonPackage { - name = "ptyprocess-0.5.2"; - buildInputs = with self; []; + "ptyprocess" = super.buildPythonPackage { + name = "ptyprocess-0.6.0"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/51/83/5d07dc35534640b06f9d9f1a1d2bc2513fb9cc7595a1b0e28ae5477056ce/ptyprocess-0.5.2.tar.gz"; - md5 = "d3b8febae1b8c53b054bd818d0bb8665"; + url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz"; + sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj"; }; meta = { license = [ ]; }; }; - py = super.buildPythonPackage { - name = "py-1.5.2"; - buildInputs = with self; []; + "py" = super.buildPythonPackage { + name = "py-1.5.3"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/90/e3/e075127d39d35f09a500ebb4a90afd10f9ef0a1d28a6d09abeec0e444fdd/py-1.5.2.tar.gz"; - md5 = "279ca69c632069e1b71e11b14641ca28"; + url = "https://files.pythonhosted.org/packages/f7/84/b4c6e84672c4ceb94f727f3da8344037b62cee960d80e999b1cd9b832d83/py-1.5.3.tar.gz"; + sha256 = "10gq2lckvgwlk9w6yzijhzkarx44hsaknd0ypa08wlnpjnsgmj99"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - pygments = super.buildPythonPackage { + "pygments" = super.buildPythonPackage { name = "pygments-2.2.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz"; - md5 = "13037baca42f16917cbd5ad2fab50844"; + url = "https://files.pythonhosted.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz"; + sha256 = "1k78qdvir1yb1c634nkv6rbga8wv4289xarghmsbbvzhvr311bnv"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - pyramid = super.buildPythonPackage { - name = "pyramid-1.9.1"; - buildInputs = with self; []; + "pyramid" = super.buildPythonPackage { + name = "pyramid-1.9.2"; doCheck = false; - propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy plaster plaster-pastedeploy hupper]; + propagatedBuildInputs = [ + self."setuptools" + self."webob" + self."repoze.lru" + self."zope.interface" + self."zope.deprecation" + self."venusian" + self."translationstring" + self."pastedeploy" + self."plaster" + self."plaster-pastedeploy" + self."hupper" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/9a/57/73447be9e7d0512d601e3f0a1fb9d7d1efb941911f49efdfe036d2826507/pyramid-1.9.1.tar.gz"; - md5 = "0163e19c58c2d12976a3b6fdb57e052d"; + url = "https://files.pythonhosted.org/packages/a0/c1/b321d07cfc4870541989ad131c86a1d593bfe802af0eca9718a0dadfb97a/pyramid-1.9.2.tar.gz"; + sha256 = "09drsl0346nchgxp2j7sa5hlk7mkhfld9wvbd0wicacrp26a92fg"; }; meta = { license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; }; }; - pyramid-jinja2 = super.buildPythonPackage { - name = "pyramid-jinja2-2.7"; - buildInputs = with self; []; + "pyramid-mako" = super.buildPythonPackage { + name = "pyramid-mako-1.0.2"; doCheck = false; - propagatedBuildInputs = with self; [pyramid zope.deprecation Jinja2 MarkupSafe]; + propagatedBuildInputs = [ + self."pyramid" + self."mako" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/d8/80/d60a7233823de22ce77bd864a8a83736a1fe8b49884b08303a2e68b2c853/pyramid_jinja2-2.7.tar.gz"; - md5 = "c2f8b2cd7b73a6f1d9a311fcfaf4fb92"; + url = "https://files.pythonhosted.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz"; + sha256 = "18gk2vliq8z4acblsl6yzgbvnr9rlxjlcqir47km7kvlk1xri83d"; }; meta = { license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; }; }; - pyramid-mako = super.buildPythonPackage { - name = "pyramid-mako-1.0.2"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [pyramid Mako]; - src = fetchurl { - url = "https://pypi.python.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz"; - md5 = "ee25343a97eb76bd90abdc2a774eb48a"; - }; - meta = { - license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; - }; - }; - pytest = super.buildPythonPackage { - name = "pytest-3.2.5"; - buildInputs = with self; []; + "pytest" = super.buildPythonPackage { + name = "pytest-3.6.0"; doCheck = false; - propagatedBuildInputs = with self; [py setuptools]; + propagatedBuildInputs = [ + self."py" + self."six" + self."setuptools" + self."attrs" + self."more-itertools" + self."atomicwrites" + self."pluggy" + self."funcsigs" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/1f/f8/8cd74c16952163ce0db0bd95fdd8810cbf093c08be00e6e665ebf0dc3138/pytest-3.2.5.tar.gz"; - md5 = "6dbe9bb093883f75394a689a1426ac6f"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; - pytest-catchlog = super.buildPythonPackage { - name = "pytest-catchlog-1.2.2"; - buildInputs = with self; []; - doCheck = false; - propagatedBuildInputs = with self; [py pytest]; - src = fetchurl { - url = "https://pypi.python.org/packages/f2/2b/2faccdb1a978fab9dd0bf31cca9f6847fbe9184a0bdcc3011ac41dd44191/pytest-catchlog-1.2.2.zip"; - md5 = "09d890c54c7456c818102b7ff8c182c8"; + url = "https://files.pythonhosted.org/packages/67/6a/5bcdc22f8dbada1d2910d6e1a3a03f6b14306c78f81122890735b28be4bf/pytest-3.6.0.tar.gz"; + sha256 = "0bdfazvjjbxssqzyvkb3m2x2in7xv56ipr899l00s87k7815sm9r"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - pytest-cov = super.buildPythonPackage { + "pytest-cov" = super.buildPythonPackage { name = "pytest-cov-2.5.1"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; [pytest coverage]; + propagatedBuildInputs = [ + self."pytest" + self."coverage" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/24/b4/7290d65b2f3633db51393bdf8ae66309b37620bc3ec116c5e357e3e37238/pytest-cov-2.5.1.tar.gz"; - md5 = "5acf38d4909e19819eb5c1754fbfc0ac"; + url = "https://files.pythonhosted.org/packages/24/b4/7290d65b2f3633db51393bdf8ae66309b37620bc3ec116c5e357e3e37238/pytest-cov-2.5.1.tar.gz"; + sha256 = "0bbfpwdh9k3636bxc88vz9fa7vf4akchgn513ql1vd0xy4n7bah3"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ]; }; }; - pytest-profiling = super.buildPythonPackage { - name = "pytest-profiling-1.2.11"; - buildInputs = with self; []; + "pytest-profiling" = super.buildPythonPackage { + name = "pytest-profiling-1.3.0"; doCheck = false; - propagatedBuildInputs = with self; [six pytest gprof2dot]; + propagatedBuildInputs = [ + self."six" + self."pytest" + self."gprof2dot" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/c0/4a/b4aa786e93c07a86f1f87c581a36bf355a9e06a9da7e00dbd05047626bd2/pytest-profiling-1.2.11.tar.gz"; - md5 = "9ef6b60248731be5d44477980408e8f7"; + url = "https://files.pythonhosted.org/packages/f5/34/4626126e041a51ef50a80d0619519b18d20aef249aac25b0d0fdd47e57ee/pytest-profiling-1.3.0.tar.gz"; + sha256 = "08r5afx5z22yvpmsnl91l4amsy1yxn8qsmm61mhp06mz8zjs51kb"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - pytest-runner = super.buildPythonPackage { - name = "pytest-runner-3.0"; - buildInputs = with self; []; + "pytest-runner" = super.buildPythonPackage { + name = "pytest-runner-4.2"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/65/b4/ae89338cd2d81e2cc54bd6db2e962bfe948f612303610d68ab24539ac2d1/pytest-runner-3.0.tar.gz"; - md5 = "8f8363a52bbabc4cedd5e239beb2ba11"; + url = "https://files.pythonhosted.org/packages/9e/b7/fe6e8f87f9a756fd06722216f1b6698ccba4d269eac6329d9f0c441d0f93/pytest-runner-4.2.tar.gz"; + sha256 = "1gkpyphawxz38ni1gdq1fmwyqcg02m7ypzqvv46z06crwdxi2gyj"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - pytest-sugar = super.buildPythonPackage { - name = "pytest-sugar-0.9.0"; - buildInputs = with self; []; + "pytest-sugar" = super.buildPythonPackage { + name = "pytest-sugar-0.9.1"; doCheck = false; - propagatedBuildInputs = with self; [pytest termcolor]; + propagatedBuildInputs = [ + self."pytest" + self."termcolor" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/49/d8/c5ff6cca3ce2ebd8b73eec89779bf6b4a7737456a70e8ea4d44c1ff90f71/pytest-sugar-0.9.0.tar.gz"; - md5 = "89fbff17277fa6a95a560a04b68cb9f9"; + url = "https://files.pythonhosted.org/packages/3e/6a/a3f909083079d03bde11d06ab23088886bbe25f2c97fbe4bb865e2bf05bc/pytest-sugar-0.9.1.tar.gz"; + sha256 = "0b4av40dv30727m54v211r0nzwjp2ajkjgxix6j484qjmwpw935b"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - pytest-timeout = super.buildPythonPackage { - name = "pytest-timeout-1.2.0"; - buildInputs = with self; []; + "pytest-timeout" = super.buildPythonPackage { + name = "pytest-timeout-1.2.1"; doCheck = false; - propagatedBuildInputs = with self; [pytest]; + propagatedBuildInputs = [ + self."pytest" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/cc/b7/b2a61365ea6b6d2e8881360ae7ed8dad0327ad2df89f2f0be4a02304deb2/pytest-timeout-1.2.0.tar.gz"; - md5 = "83607d91aa163562c7ee835da57d061d"; + url = "https://files.pythonhosted.org/packages/be/e9/a9106b8bc87521c6813060f50f7d1fdc15665bc1bbbe71c0ffc1c571aaa2/pytest-timeout-1.2.1.tar.gz"; + sha256 = "1kdp6qbh5v1168l99rba5yfzvy05gmzkmkhldgp36p9xcdjd5dv8"; }; meta = { license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ]; }; }; - repoze.lru = super.buildPythonPackage { + "repoze.lru" = super.buildPythonPackage { name = "repoze.lru-0.7"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz"; - md5 = "c08cc030387e0b1fc53c5c7d964b35e2"; + url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz"; + sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84"; }; meta = { license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; }; }; - rhodecode-vcsserver = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.12.4"; - buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; + "rhodecode-vcsserver" = super.buildPythonPackage { + name = "rhodecode-vcsserver-4.13.0"; + buildInputs = [ + self."pytest" + self."py" + self."pytest-cov" + self."pytest-sugar" + self."pytest-runner" + self."pytest-profiling" + self."gprof2dot" + self."pytest-timeout" + self."mock" + self."webtest" + self."cov-core" + self."coverage" + self."configobj" + ]; doCheck = true; - propagatedBuildInputs = with self; [Beaker configobj decorator dulwich hgsubversion hg-evolve infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; + propagatedBuildInputs = [ + self."configobj" + self."dogpile.cache" + self."dogpile.core" + self."decorator" + self."dulwich" + self."hgsubversion" + self."hg-evolve" + self."mako" + self."markupsafe" + self."mercurial" + self."msgpack-python" + self."pastedeploy" + self."psutil" + self."pyramid" + self."pyramid-mako" + self."pygments" + self."pathlib2" + self."repoze.lru" + self."simplejson" + self."subprocess32" + self."setproctitle" + self."subvertpy" + self."six" + self."translationstring" + self."webob" + self."zope.deprecation" + self."zope.interface" + self."gevent" + self."greenlet" + self."gunicorn" + self."waitress" + self."ipdb" + self."ipython" + self."pytest" + self."py" + self."pytest-cov" + self."pytest-sugar" + self."pytest-runner" + self."pytest-profiling" + self."gprof2dot" + self."pytest-timeout" + self."mock" + self."webtest" + self."cov-core" + self."coverage" + ]; src = ./.; meta = { license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ]; }; }; - scandir = super.buildPythonPackage { - name = "scandir-1.7"; - buildInputs = with self; []; + "scandir" = super.buildPythonPackage { + name = "scandir-1.9.0"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/13/bb/e541b74230bbf7a20a3949a2ee6631be299378a784f5445aa5d0047c192b/scandir-1.7.tar.gz"; - md5 = "037e5f24d1a0e78b17faca72dea9555f"; + url = "https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz"; + sha256 = "0r3hvf1a9jm1rkqgx40gxkmccknkaiqjavs8lccgq9s8khh5x5s4"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ]; }; }; - setuptools = super.buildPythonPackage { - name = "setuptools-30.1.0"; - buildInputs = with self; []; + "setproctitle" = super.buildPythonPackage { + name = "setproctitle-1.1.10"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/1e/43/002c8616db9a3e7be23c2556e39b90a32bb40ba0dc652de1999d5334d372/setuptools-30.1.0.tar.gz"; - md5 = "cac497f42e5096ac8df29e38d3f81c3e"; + url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz"; + sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2"; + }; + meta = { + license = [ pkgs.lib.licenses.bsdOriginal ]; + }; + }; + "setuptools" = super.buildPythonPackage { + name = "setuptools-40.1.0"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/5a/df/b2e3d9693bb0dcbeac516a73dd7a9eb82b126ae52e4a74605a9b01beddd5/setuptools-40.1.0.zip"; + sha256 = "0w1blx5ajga5y15dci0mddk49cf2xpq0mp7rp7jrqr2diqk00ib6"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - simplegeneric = super.buildPythonPackage { + "simplegeneric" = super.buildPythonPackage { name = "simplegeneric-0.8.1"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"; - md5 = "f9c1fab00fd981be588fc32759f474e3"; + url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"; + sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw"; }; meta = { - license = [ pkgs.lib.licenses.zpt21 ]; + license = [ pkgs.lib.licenses.zpl21 ]; }; }; - simplejson = super.buildPythonPackage { + "simplejson" = super.buildPythonPackage { name = "simplejson-3.11.1"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/08/48/c97b668d6da7d7bebe7ea1817a6f76394b0ec959cb04214ca833c34359df/simplejson-3.11.1.tar.gz"; - md5 = "6e2f1bd5fb0a926facf5d89d217a7183"; + url = "https://files.pythonhosted.org/packages/08/48/c97b668d6da7d7bebe7ea1817a6f76394b0ec959cb04214ca833c34359df/simplejson-3.11.1.tar.gz"; + sha256 = "1rr58dppsq73p0qcd9bsw066cdd3v63sqv7j6sqni8frvm4jv8h1"; }; meta = { license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ]; }; }; - six = super.buildPythonPackage { + "six" = super.buildPythonPackage { name = "six-1.11.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz"; - md5 = "d12789f9baf7e9fb2524c0c64f1773f8"; + url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz"; + sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - subprocess32 = super.buildPythonPackage { - name = "subprocess32-3.2.7"; - buildInputs = with self; []; + "subprocess32" = super.buildPythonPackage { + name = "subprocess32-3.5.1"; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/b8/2f/49e53b0d0e94611a2dc624a1ad24d41b6d94d0f1b0a078443407ea2214c2/subprocess32-3.2.7.tar.gz"; - md5 = "824c801e479d3e916879aae3e9c15e16"; + url = "https://files.pythonhosted.org/packages/de/fb/fd3e91507021e2aecdb081d1b920082628d6b8869ead845e3e87b3d2e2ca/subprocess32-3.5.1.tar.gz"; + sha256 = "0wgi3bfnssid1g6h0v803z3k1wjal6il16nr3r9c587cfzwfkv0q"; }; meta = { license = [ pkgs.lib.licenses.psfl ]; }; }; - subvertpy = super.buildPythonPackage { + "subvertpy" = super.buildPythonPackage { name = "subvertpy-0.10.1"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/9d/76/99fa82affce75f5ac0f7dbe513796c3f37311ace0c68e1b063683b4f9b99/subvertpy-0.10.1.tar.gz"; - md5 = "a70e03579902d480f5e9f8c570f6536b"; + url = "https://files.pythonhosted.org/packages/9d/76/99fa82affce75f5ac0f7dbe513796c3f37311ace0c68e1b063683b4f9b99/subvertpy-0.10.1.tar.gz"; + sha256 = "061ncy9wjz3zyv527avcrdyk0xygyssyy7p1644nhzhwp8zpybij"; }; meta = { license = [ pkgs.lib.licenses.lgpl21Plus pkgs.lib.licenses.gpl2Plus ]; }; }; - termcolor = super.buildPythonPackage { + "termcolor" = super.buildPythonPackage { name = "termcolor-1.1.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz"; - md5 = "043e89644f8909d462fbbfa511c768df"; + url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz"; + sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - traitlets = super.buildPythonPackage { + "traitlets" = super.buildPythonPackage { name = "traitlets-4.3.2"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; [ipython-genutils six decorator enum34]; + propagatedBuildInputs = [ + self."ipython-genutils" + self."six" + self."decorator" + self."enum34" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/a5/98/7f5ef2fe9e9e071813aaf9cb91d1a732e0a68b6c44a32b38cb8e14c3f069/traitlets-4.3.2.tar.gz"; - md5 = "3068663f2f38fd939a9eb3a500ccc154"; + url = "https://files.pythonhosted.org/packages/a5/98/7f5ef2fe9e9e071813aaf9cb91d1a732e0a68b6c44a32b38cb8e14c3f069/traitlets-4.3.2.tar.gz"; + sha256 = "0dbq7sx26xqz5ixs711k5nc88p8a0nqyz6162pwks5dpcz9d4jww"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - translationstring = super.buildPythonPackage { + "translationstring" = super.buildPythonPackage { name = "translationstring-1.3"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz"; - md5 = "a4b62e0f3c189c783a1685b3027f7c90"; + url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz"; + sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f"; }; meta = { license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ]; }; }; - venusian = super.buildPythonPackage { + "venusian" = super.buildPythonPackage { name = "venusian-1.1.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/38/24/b4b470ab9e0a2e2e9b9030c7735828c8934b4c6b45befd1bb713ec2aeb2d/venusian-1.1.0.tar.gz"; - md5 = "56bc5e6756e4bda37bcdb94f74a72b8f"; + url = "https://files.pythonhosted.org/packages/38/24/b4b470ab9e0a2e2e9b9030c7735828c8934b4c6b45befd1bb713ec2aeb2d/venusian-1.1.0.tar.gz"; + sha256 = "0zapz131686qm0gazwy8bh11vr57pr89jbwbl50s528sqy9f80lr"; }; meta = { license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; }; }; - waitress = super.buildPythonPackage { + "waitress" = super.buildPythonPackage { name = "waitress-1.1.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/3c/68/1c10dd5c556872ceebe88483b0436140048d39de83a84a06a8baa8136f4f/waitress-1.1.0.tar.gz"; - md5 = "0f1eb7fdfdbf2e6d18decbda1733045c"; + url = "https://files.pythonhosted.org/packages/3c/68/1c10dd5c556872ceebe88483b0436140048d39de83a84a06a8baa8136f4f/waitress-1.1.0.tar.gz"; + sha256 = "1a85gyji0kajc3p0s1pwwfm06w4wfxjkvvl4rnrz3h164kbd6g6k"; }; meta = { - license = [ pkgs.lib.licenses.zpt21 ]; + license = [ pkgs.lib.licenses.zpl21 ]; }; }; - wcwidth = super.buildPythonPackage { + "wcwidth" = super.buildPythonPackage { name = "wcwidth-0.1.7"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz"; - md5 = "b3b6a0a08f0c8a34d1de8cf44150a4ad"; + url = "https://files.pythonhosted.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz"; + sha256 = "0pn6dflzm609m4r3i8ik5ni9ijjbb5fa3vg1n7hn6vkd49r77wrx"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; + "webob" = super.buildPythonPackage { + name = "webob-1.7.4"; + doCheck = false; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/75/34/731e23f52371852dfe7490a61644826ba7fe70fd52a377aaca0f4956ba7f/WebOb-1.7.4.tar.gz"; + sha256 = "1na01ljg04z40il7vcrn8g29vaw7nvg1xvhk64cr4jys5wcay44d"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - wheel = super.buildPythonPackage { - name = "wheel-0.29.0"; - buildInputs = with self; []; + "webtest" = super.buildPythonPackage { + name = "webtest-2.0.29"; doCheck = false; - propagatedBuildInputs = with self; []; + propagatedBuildInputs = [ + self."six" + self."webob" + self."waitress" + self."beautifulsoup4" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz"; - md5 = "555a67e4507cedee23a0deb9651e452f"; + url = "https://files.pythonhosted.org/packages/94/de/8f94738be649997da99c47b104aa3c3984ecec51a1d8153ed09638253d56/WebTest-2.0.29.tar.gz"; + sha256 = "0bcj1ica5lnmj5zbvk46x28kgphcsgh7sfnwjmn0cr94mhawrg6v"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; - zope.deprecation = super.buildPythonPackage { + "zope.deprecation" = super.buildPythonPackage { name = "zope.deprecation-4.3.0"; - buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; [setuptools]; + propagatedBuildInputs = [ + self."setuptools" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz"; - md5 = "2166b2cb7e0e96a21104e6f8f9b696bb"; + url = "https://files.pythonhosted.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz"; + sha256 = "095jas41wbxgmw95kwdxqhbc3bgihw2hzj9b3qpdg85apcsf2lkx"; }; meta = { - license = [ pkgs.lib.licenses.zpt21 ]; + license = [ pkgs.lib.licenses.zpl21 ]; }; }; - zope.interface = super.buildPythonPackage { - name = "zope.interface-4.4.3"; - buildInputs = with self; []; + "zope.interface" = super.buildPythonPackage { + name = "zope.interface-4.5.0"; doCheck = false; - propagatedBuildInputs = with self; [setuptools]; + propagatedBuildInputs = [ + self."setuptools" + ]; src = fetchurl { - url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; - md5 = "8700a4f527c1203b34b10c2b4e7a6912"; + url = "https://files.pythonhosted.org/packages/ac/8a/657532df378c2cd2a1fe6b12be3b4097521570769d4852ec02c24bd3594e/zope.interface-4.5.0.tar.gz"; + sha256 = "0k67m60ij06wkg82n15qgyn96waf4pmrkhv0njpkfzpmv5q89hsp"; }; meta = { - license = [ pkgs.lib.licenses.zpt21 ]; + license = [ pkgs.lib.licenses.zpl21 ]; }; }; diff --git a/pkgs/shell-generate.nix b/pkgs/shell-generate.nix new file mode 100755 --- /dev/null +++ b/pkgs/shell-generate.nix @@ -0,0 +1,41 @@ +{ pkgs ? (import {}) +, pythonPackages ? "python27Packages" +}: + +with pkgs.lib; + +let _pythonPackages = pythonPackages; in +let + pythonPackages = getAttr _pythonPackages pkgs; + + pip2nix = import ./nix-common/pip2nix.nix { + inherit + pkgs + pythonPackages; + }; + +in + +pkgs.stdenv.mkDerivation { + name = "pip2nix-generated"; + buildInputs = [ + pip2nix.pip2nix + pythonPackages.pip-tools + pkgs.apr + pkgs.aprutil + ]; + + shellHook = '' + runHook preShellHook + echo "Setting SVN_* variables" + export SVN_LIBRARY_PATH=${pkgs.subversion}/lib + export SVN_HEADER_PATH=${pkgs.subversion.dev}/include + runHook postShellHook + ''; + + preShellHook = '' + echo "Starting Generate Shell" + # Custom prompt to distinguish from other dev envs. + export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] " + ''; +} diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,8 @@ [pytest] -testpaths = ./vcsserver -addopts = -v +testpaths = vcsserver +norecursedirs = vcsserver/hook_utils/hook_templates +cache_dir = /tmp/.pytest_cache + + +addopts = + --pdbcls=IPython.terminal.debugger:TerminalPdb diff --git a/release.nix b/release.nix --- a/release.nix +++ b/release.nix @@ -1,9 +1,10 @@ +# This file defines how to "build" for packaging. + { pkgs ? import {} , doCheck ? true }: let - vcsserver = import ./default.nix { inherit doCheck diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,45 @@ -## core -setuptools==30.1.0 +## dependencies -Beaker==1.9.1 -configobj==5.0.6 +# our custom configobj +https://code.rhodecode.com/upstream/configobj/archive/a11ff0a0bd4fbda9e3a91267e720f88329efb4a6.tar.gz?md5=9916c524ea11a6c418217af6b28d4b3c#egg=configobj==5.0.6 +dogpile.cache==0.6.6 +dogpile.core==0.4.1 decorator==4.1.2 dulwich==0.13.0 -hgsubversion==1.9.0 -hg-evolve==7.0.1 -infrae.cache==1.0.1 -mercurial==4.4.2 -msgpack-python==0.4.8 -pyramid-jinja2==2.7 -Jinja2==2.9.6 -pyramid==1.9.1 +hgsubversion==1.9.2 +hg-evolve==8.0.1 +mako==1.0.7 +markupsafe==1.0.0 +mercurial==4.6.2 +msgpack-python==0.5.6 + +pastedeploy==1.5.2 +psutil==5.4.6 +pyramid==1.9.2 pyramid-mako==1.0.2 + +pygments==2.2.0 +pathlib2==2.3.0 repoze.lru==0.7 simplejson==3.11.1 -subprocess32==3.2.7 - +subprocess32==3.5.1 +setproctitle==1.1.10 subvertpy==0.10.1 six==1.11.0 translationstring==1.3 -WebOb==1.7.4 -wheel==0.29.0 +webob==1.7.4 zope.deprecation==4.3.0 -zope.interface==4.4.3 +zope.interface==4.5.0 ## http servers -gevent==1.2.2 +gevent==1.3.5 greenlet==0.4.13 -gunicorn==19.7.1 +gunicorn==19.9.0 waitress==1.1.0 ## debug -ipdb==0.10.3 +ipdb==0.11.0 ipython==5.1.0 ## test related requirements diff --git a/requirements_test.txt b/requirements_test.txt --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,15 +1,14 @@ # test related requirements -pytest==3.2.5 -py==1.5.2 +pytest==3.6.0 +py==1.5.3 pytest-cov==2.5.1 -pytest-sugar==0.9.0 -pytest-runner==3.0.0 -pytest-catchlog==1.2.2 -pytest-profiling==1.2.11 +pytest-sugar==0.9.1 +pytest-runner==4.2.0 +pytest-profiling==1.3.0 gprof2dot==2017.9.19 -pytest-timeout==1.2.0 +pytest-timeout==1.2.1 mock==1.0.1 -WebTest==2.0.29 +webtest==2.0.29 cov-core==1.15.0 coverage==3.7.1 diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -23,11 +23,18 @@ import os import sys import pkgutil import platform +import codecs -from pip.download import PipSession -from pip.req import parse_requirements +try: # for pip >= 10 + from pip._internal.req import parse_requirements +except ImportError: # for pip <= 9.0.3 + from pip.req import parse_requirements -from codecs import open +try: # for pip >= 10 + from pip._internal.download import PipSession +except ImportError: # for pip <= 9.0.3 + from pip.download import PipSession + if sys.version_info < (2, 7): @@ -89,8 +96,8 @@ keywords = ' '.join([ readme_file = 'README.rst' changelog_file = 'CHANGES.rst' try: - long_description = open(readme_file).read() + '\n\n' + \ - open(changelog_file).read() + long_description = codecs.open(readme_file).read() + '\n\n' + \ + codecs.open(changelog_file).read() except IOError as err: sys.stderr.write( "[WARNING] Cannot find file specified as long_description (%s)\n " @@ -106,7 +113,7 @@ setup( keywords=keywords, license=__license__, author=__author__, - author_email='marcin@rhodecode.com', + author_email='admin@rhodecode.com', url=__url__, setup_requires=setup_requirements, install_requires=install_requirements, diff --git a/shell.nix b/shell.nix --- a/shell.nix +++ b/shell.nix @@ -1,41 +1,67 @@ -{ pkgs ? import {} +# This file contains the adjustments which are desired for a development +# environment. + +{ pkgs ? (import {}) +, pythonPackages ? "python27Packages" , doCheck ? false }: let vcsserver = import ./default.nix { - inherit pkgs doCheck; + inherit + pkgs + doCheck; }; vcs-pythonPackages = vcsserver.pythonPackages; in vcsserver.override (attrs: { - # Avoid that we dump any sources into the store when entering the shell and # make development a little bit more convenient. src = null; + # Add dependencies which are useful for the development environment. buildInputs = attrs.buildInputs ++ (with vcs-pythonPackages; [ ipdb ]); - # Somewhat snappier setup of the development environment - # TODO: think of supporting a stable path again, so that multiple shells - # can share it. - postShellHook = '' - # Set locale - export LC_ALL="en_US.UTF-8" + # place to inject some required libs from develop installs + propagatedBuildInputs = + attrs.propagatedBuildInputs ++ + []; + + + # Make sure we execute both hooks + shellHook = '' + runHook preShellHook + runHook postShellHook + ''; + + preShellHook = '' + echo "Entering VCS-Shell" # Custom prompt to distinguish from other dev envs. export PS1="\n\[\033[1;32m\][VCS-shell:\w]$\[\033[0m\] " + # Set locale + export LC_ALL="en_US.UTF-8" + + # Setup a temporary directory. tmp_path=$(mktemp -d) export PATH="$tmp_path/bin:$PATH" export PYTHONPATH="$tmp_path/${vcs-pythonPackages.python.sitePackages}:$PYTHONPATH" mkdir -p $tmp_path/${vcs-pythonPackages.python.sitePackages} + + # Develop installation + echo "[BEGIN]: develop install of rhodecode-vcsserver" python setup.py develop --prefix $tmp_path --allow-hosts "" ''; + + postShellHook = '' + + ''; + }) diff --git a/test.ini b/test.ini deleted file mode 100644 --- a/test.ini +++ /dev/null @@ -1,87 +0,0 @@ -################################################################################ -# RhodeCode VCSServer - configuration # -# # -################################################################################ - -[DEFAULT] -host = 127.0.0.1 -port = 9901 -locale = en_US.UTF-8 -# number of worker threads, this should be set based on a formula threadpool=N*6 -# where N is number of RhodeCode Enterprise workers, eg. running 2 instances -# 8 gunicorn workers each would be 2 * 8 * 6 = 96, threadpool_size = 96 -threadpool_size = 96 -timeout = 0 - -# cache regions, please don't change -beaker.cache.regions = repo_object -beaker.cache.repo_object.type = memorylru -beaker.cache.repo_object.max_items = 100 -# cache auto-expires after N seconds -beaker.cache.repo_object.expire = 300 -beaker.cache.repo_object.enabled = true - - -################################ -### LOGGING CONFIGURATION #### -################################ -[loggers] -keys = root, vcsserver, beaker - -[handlers] -keys = console - -[formatters] -keys = generic - -############# -## LOGGERS ## -############# -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker -propagate = 1 - - -############## -## HANDLERS ## -############## - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = INFO -formatter = generic - -[handler_file] -class = FileHandler -args = ('vcsserver.log', 'a',) -level = DEBUG -formatter = generic - -[handler_file_rotating] -class = logging.handlers.TimedRotatingFileHandler -# 'D', 5 - rotate every 5days -# you can set 'h', 'midnight' -args = ('vcsserver.log', 'D', 5, 10,) -level = DEBUG -formatter = generic - -################ -## FORMATTERS ## -################ - -[formatter_generic] -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.12.4 \ No newline at end of file +4.13.0 \ No newline at end of file diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -20,6 +20,7 @@ import traceback import logging import urlparse +from vcsserver.lib.rc_cache import region_meta log = logging.getLogger(__name__) @@ -30,9 +31,10 @@ class RepoFactory(object): It provides internal caching of the `repo` object based on the :term:`call context`. """ + repo_type = None - def __init__(self, repo_cache): - self._cache = repo_cache + def __init__(self): + self._cache_region = region_meta.dogpile_cache_regions['repo_object'] def _create_config(self, path, config): config = {} @@ -48,26 +50,19 @@ class RepoFactory(object): Uses internally the low level beaker API since the decorators introduce significant overhead. """ - def create_new_repo(): + region = self._cache_region + context = wire.get('context', None) + repo_path = wire.get('path', '') + context_uid = '{}'.format(context) + cache = wire.get('cache', True) + cache_on = context and cache + + @region.conditional_cache_on_arguments(condition=cache_on) + def create_new_repo(_repo_type, _repo_path, _context_uid): return self._create_repo(wire, create) - return self._repo(wire, create_new_repo) - - def _repo(self, wire, createfunc): - context = wire.get('context', None) - cache = wire.get('cache', True) - - if context and cache: - cache_key = (context, wire['path']) - log.debug( - 'FETCH %s@%s repo object from cache. Context: %s', - self.__class__.__name__, wire['path'], context) - return self._cache.get(key=cache_key, createfunc=createfunc) - else: - log.debug( - 'INIT %s@%s repo object based on wire %s. Context: %s', - self.__class__.__name__, wire['path'], wire, context) - return createfunc() + repo = create_new_repo(self.repo_type, repo_path, context_uid) + return repo def obfuscate_qs(query_string): @@ -90,8 +85,6 @@ def raise_from_original(new_type): """ exc_type, exc_value, exc_traceback = sys.exc_info() - traceback.format_exception(exc_type, exc_value, exc_traceback) - try: raise new_type(*exc_value.args), None, exc_traceback finally: diff --git a/vcsserver/exceptions.py b/vcsserver/exceptions.py --- a/vcsserver/exceptions.py +++ b/vcsserver/exceptions.py @@ -24,11 +24,10 @@ which contain an extra attribute `_vcs_k different error conditions. """ -import functools -from pyramid.httpexceptions import HTTPLocked +from pyramid.httpexceptions import HTTPLocked, HTTPForbidden -def _make_exception(kind, *args): +def _make_exception(kind, org_exc, *args): """ Prepares a base `Exception` instance to be sent over the wire. @@ -37,26 +36,68 @@ def _make_exception(kind, *args): """ exc = Exception(*args) exc._vcs_kind = kind + exc._org_exc = org_exc return exc -AbortException = functools.partial(_make_exception, 'abort') +def AbortException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('abort', org_exc, *args) + return _make_exception_wrapper + -ArchiveException = functools.partial(_make_exception, 'archive') +def ArchiveException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('archive', org_exc, *args) + return _make_exception_wrapper + -LookupException = functools.partial(_make_exception, 'lookup') +def LookupException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('lookup', org_exc, *args) + return _make_exception_wrapper + -VcsException = functools.partial(_make_exception, 'error') +def VcsException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('error', org_exc, *args) + return _make_exception_wrapper + + +def RepositoryLockedException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('repo_locked', org_exc, *args) + return _make_exception_wrapper -RepositoryLockedException = functools.partial(_make_exception, 'repo_locked') + +def RepositoryBranchProtectedException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('repo_branch_protected', org_exc, *args) + return _make_exception_wrapper -RequirementException = functools.partial(_make_exception, 'requirement') + +def RequirementException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('requirement', org_exc, *args) + return _make_exception_wrapper + -UnhandledException = functools.partial(_make_exception, 'unhandled') +def UnhandledException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('unhandled', org_exc, *args) + return _make_exception_wrapper + -URLError = functools.partial(_make_exception, 'url_error') +def URLError(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('url_error', org_exc, *args) + return _make_exception_wrapper -SubrepoMergeException = functools.partial(_make_exception, 'subrepo_merge_error') + +def SubrepoMergeException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('subrepo_merge_error', org_exc, *args) + return _make_exception_wrapper class HTTPRepoLocked(HTTPLocked): @@ -68,3 +109,8 @@ class HTTPRepoLocked(HTTPLocked): self.code = status_code or HTTPLocked.code self.title = title super(HTTPRepoLocked, self).__init__(**kwargs) + + +class HTTPRepoBranchProtected(HTTPForbidden): + def __init__(self, *args, **kwargs): + super(HTTPForbidden, self).__init__(*args, **kwargs) diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -56,9 +56,9 @@ def reraise_safe_exceptions(func): return func(*args, **kwargs) except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing) as e: - raise exceptions.LookupException(e.message) + raise exceptions.LookupException(e)(e.message) except (HangupException, UnexpectedCommandError) as e: - raise exceptions.VcsException(e.message) + raise exceptions.VcsException(e)(e.message) except Exception as e: # NOTE(marcink): becuase of how dulwich handles some exceptions # (KeyError on empty repos), we cannot track this and catch all @@ -87,6 +87,7 @@ class Repo(DulwichRepo): class GitFactory(RepoFactory): + repo_type = 'git' def _create_repo(self, wire, create): repo_path = str_to_dulwich(wire['path']) @@ -213,8 +214,8 @@ class GitRemote(object): elif attr in ["author", "message", "parents"]: args.append(attr) result[attr] = method(*args) - except KeyError: - raise exceptions.VcsException( + except KeyError as e: + raise exceptions.VcsException(e)( "Unknown bulk attribute: %s" % attr) return result @@ -257,11 +258,11 @@ class GitRemote(object): log.debug("Trying to open URL %s", cleaned_uri) resp = o.open(req) if resp.code != 200: - raise exceptions.URLError('Return Code is not 200') + raise exceptions.URLError()('Return Code is not 200') except Exception as e: log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True) # means it cannot be cloned - raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e)) + raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e)) # now detect if it's proper git repo gitdata = resp.read() @@ -271,7 +272,7 @@ class GitRemote(object): # old style git can return some other format ! pass else: - raise exceptions.URLError( + raise exceptions.URLError()( "url [%s] does not look like an git" % (cleaned_uri,)) return True @@ -418,7 +419,7 @@ class GitRemote(object): log.warning( 'Trying to fetch from "%s" failed, not a Git repository.', url) # Exception can contain unicode which we convert - raise exceptions.AbortException(repr(e)) + raise exceptions.AbortException(e)(repr(e)) # mikhail: client.fetch() returns all the remote refs, but fetches only # refs filtered by `determine_wants` function. We need to filter result @@ -525,9 +526,13 @@ class GitRemote(object): return repo.refs.path @reraise_safe_exceptions - def head(self, wire): + def head(self, wire, show_exc=True): repo = self._factory.repo(wire) - return repo.head() + try: + return repo.head() + except Exception: + if show_exc: + raise @reraise_safe_exceptions def init(self, wire): @@ -654,7 +659,7 @@ class GitRemote(object): if safe_call: return '', err else: - raise exceptions.VcsException(tb_err) + raise exceptions.VcsException()(tb_err) @reraise_safe_exceptions def install_hooks(self, wire, force=False): diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -32,7 +32,7 @@ from vcsserver.base import RepoFactory, from vcsserver.hgcompat import ( archival, bin, clone, config as hgconfig, diffopts, hex, hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler, - httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev, + makepeer, localrepository, match, memctx, exchange, memfilectx, nullrev, patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError, RepoLookupError, InterventionRequired, RequirementError) @@ -74,25 +74,27 @@ def reraise_safe_exceptions(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except (Abort, InterventionRequired): - raise_from_original(exceptions.AbortException) - except RepoLookupError: - raise_from_original(exceptions.LookupException) - except RequirementError: - raise_from_original(exceptions.RequirementException) - except RepoError: - raise_from_original(exceptions.VcsException) - except LookupError: - raise_from_original(exceptions.LookupException) + except (Abort, InterventionRequired) as e: + raise_from_original(exceptions.AbortException(e)) + except RepoLookupError as e: + raise_from_original(exceptions.LookupException(e)) + except RequirementError as e: + raise_from_original(exceptions.RequirementException(e)) + except RepoError as e: + raise_from_original(exceptions.VcsException(e)) + except LookupError as e: + raise_from_original(exceptions.LookupException(e)) except Exception as e: if not hasattr(e, '_vcs_kind'): log.exception("Unhandled exception in hg remote call") - raise_from_original(exceptions.UnhandledException) + raise_from_original(exceptions.UnhandledException(e)) + raise return wrapper class MercurialFactory(RepoFactory): + repo_type = 'hg' def _create_config(self, config, hooks=True): if not hooks: @@ -148,7 +150,7 @@ class HgRemote(object): elif kind == 'zip': archiver = archival.zipit(archive_path, mtime) else: - raise exceptions.ArchiveException( + raise exceptions.ArchiveException()( 'Remote does not support: "%s".' % kind) for f_path, f_mode, f_is_link, f_content in file_info: @@ -180,8 +182,8 @@ class HgRemote(object): try: method = self._bulk_methods[attr] result[attr] = method(wire, rev) - except KeyError: - raise exceptions.VcsException( + except KeyError as e: + raise exceptions.VcsException(e)( 'Unknown bulk attribute: "%s"' % attr) return result @@ -211,14 +213,14 @@ class HgRemote(object): if node['path'] == path: return memfilectx( _repo, + changectx=memctx, path=node['path'], data=node['content'], islink=False, isexec=bool(node['mode'] & stat.S_IXUSR), - copied=False, - memctx=memctx) + copied=False) - raise exceptions.AbortException( + raise exceptions.AbortException()( "Given path haven't been marked as added, " "changed or removed (%s)" % path) @@ -368,11 +370,11 @@ class HgRemote(object): log.debug("Trying to open URL %s", cleaned_uri) resp = o.open(req) if resp.code != 200: - raise exceptions.URLError('Return Code is not 200') + raise exceptions.URLError()('Return Code is not 200') except Exception as e: log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True) # means it cannot be cloned - raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e)) + raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e)) # now check if it's a proper hg repo, but don't do it for svn try: @@ -383,11 +385,13 @@ class HgRemote(object): log.debug( "Verifying if URL is a Mercurial repository: %s", cleaned_uri) - httppeer(make_ui_from_config(config), url).lookup('tip') + ui = make_ui_from_config(config) + peer_checker = makepeer(ui, url) + peer_checker.lookup('tip') except Exception as e: log.warning("URL is not a valid Mercurial repository: %s", cleaned_uri) - raise exceptions.URLError( + raise exceptions.URLError(e)( "url [%s] does not look like an hg repo org_exc: %s" % (cleaned_uri, e)) @@ -409,8 +413,8 @@ class HgRemote(object): try: return "".join(patch.diff( repo, node1=rev1, node2=rev2, match=match_filter, opts=opts)) - except RepoLookupError: - raise exceptions.LookupException() + except RepoLookupError as e: + raise exceptions.LookupException(e)() @reraise_safe_exceptions def file_history(self, wire, revision, path, limit): @@ -454,9 +458,10 @@ class HgRemote(object): fctx = ctx.filectx(path) result = [] - for i, (a_line, content) in enumerate(fctx.annotate()): - ln_no = i + 1 - sha = hex(a_line.fctx.node()) + for i, annotate_obj in enumerate(fctx.annotate(), 1): + ln_no = i + sha = hex(annotate_obj.fctx.node()) + content = annotate_obj.text result.append((ln_no, sha, content)) return result @@ -533,16 +538,28 @@ class HgRemote(object): @reraise_safe_exceptions def lookup(self, wire, revision, both): - # TODO Paris: Ugly hack to "deserialize" long for msgpack - if isinstance(revision, float): - revision = long(revision) + repo = self._factory.repo(wire) + + if isinstance(revision, int): + # NOTE(marcink): + # since Mercurial doesn't support indexes properly + # we need to shift accordingly by one to get proper index, e.g + # repo[-1] => repo[-2] + # repo[0] => repo[-1] + # repo[1] => repo[2] we also never call repo[0] because + # it's actually second commit + if revision <= 0: + revision = revision + -1 + else: + revision = revision + 1 + try: ctx = repo[revision] - except RepoLookupError: - raise exceptions.LookupException(revision) + except RepoLookupError as e: + raise exceptions.LookupException(e)(revision) except LookupError as e: - raise exceptions.LookupException(e.name) + raise exceptions.LookupException(e)(e.name) if not both: return ctx.hex() @@ -658,7 +675,7 @@ class HgRemote(object): except Abort as e: log.exception("Tag operation aborted") # Exception can contain unicode which we convert - raise exceptions.AbortException(repr(e)) + raise exceptions.AbortException(e)(repr(e)) @reraise_safe_exceptions def tags(self, wire): diff --git a/vcsserver/hgcompat.py b/vcsserver/hgcompat.py --- a/vcsserver/hgcompat.py +++ b/vcsserver/hgcompat.py @@ -51,7 +51,7 @@ from mercurial.node import bin, hex from mercurial.encoding import tolocal from mercurial.discovery import findcommonoutgoing from mercurial.hg import peer -from mercurial.httppeer import httppeer +from mercurial.httppeer import makepeer from mercurial.util import url as hg_url from mercurial.scmutil import revrange from mercurial.node import nullrev diff --git a/vcsserver/hgpatches.py b/vcsserver/hgpatches.py --- a/vcsserver/hgpatches.py +++ b/vcsserver/hgpatches.py @@ -36,15 +36,15 @@ def patch_largefiles_capabilities(): lfproto = hgcompat.largefiles.proto wrapper = _dynamic_capabilities_wrapper( lfproto, hgcompat.extensions.extensions) - lfproto.capabilities = wrapper + lfproto._capabilities = wrapper def _dynamic_capabilities_wrapper(lfproto, extensions): - wrapped_capabilities = lfproto.capabilities + wrapped_capabilities = lfproto._capabilities logger = logging.getLogger('vcsserver.hg') - def _dynamic_capabilities(repo, proto): + def _dynamic_capabilities(orig, repo, proto): """ Adds dynamic behavior, so that the capability is only added if the extension is enabled in the current ui object. @@ -52,10 +52,10 @@ def _dynamic_capabilities_wrapper(lfprot if 'largefiles' in dict(extensions(repo.ui)): logger.debug('Extension largefiles enabled') calc_capabilities = wrapped_capabilities + return calc_capabilities(orig, repo, proto) else: logger.debug('Extension largefiles disabled') - calc_capabilities = lfproto.capabilitiesorig - return calc_capabilities(repo, proto) + return orig(repo, proto) return _dynamic_capabilities @@ -116,7 +116,7 @@ def patch_subrepo_type_mapping(): def merge(self, state): """merge currently-saved state with the new state.""" - raise SubrepoMergeException() + raise SubrepoMergeException()() def push(self, opts): """perform whatever action is analogous to 'hg push' diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -120,9 +120,11 @@ def _handle_exception(result): log.error('Got traceback from remote call:%s', exception_traceback) if exception_class == 'HTTPLockedRC': - raise exceptions.RepositoryLockedException(*result['exception_args']) + raise exceptions.RepositoryLockedException()(*result['exception_args']) + elif exception_class == 'HTTPBranchProtected': + raise exceptions.RepositoryBranchProtectedException()(*result['exception_args']) elif exception_class == 'RepositoryError': - raise exceptions.VcsException(*result['exception_args']) + raise exceptions.VcsException()(*result['exception_args']) elif exception_class: raise Exception('Got remote exception "%s" with args "%s"' % (exception_class, result['exception_args'])) @@ -161,16 +163,54 @@ def _extras_from_ui(ui): return extras -def _rev_range_hash(repo, node): +def _rev_range_hash(repo, node, check_heads=False): commits = [] - for rev in xrange(repo[node], len(repo)): + revs = [] + start = repo[node].rev() + end = len(repo) + for rev in range(start, end): + revs.append(rev) ctx = repo[rev] commit_id = mercurial.node.hex(ctx.node()) branch = ctx.branch() commits.append((commit_id, branch)) - return commits + parent_heads = [] + if check_heads: + parent_heads = _check_heads(repo, start, end, revs) + return commits, parent_heads + + +def _check_heads(repo, start, end, commits): + changelog = repo.changelog + parents = set() + + for new_rev in commits: + for p in changelog.parentrevs(new_rev): + if p == mercurial.node.nullrev: + continue + if p < start: + parents.add(p) + + for p in parents: + branch = repo[p].branch() + # The heads descending from that parent, on the same branch + parent_heads = set([p]) + reachable = set([p]) + for x in xrange(p + 1, end): + if repo[x].branch() != branch: + continue + for pp in changelog.parentrevs(x): + if pp in reachable: + reachable.add(x) + parent_heads.discard(pp) + parent_heads.add(x) + # More than one head? Suggest merging + if len(parent_heads) > 1: + return list(parent_heads) + + return [] def repo_size(ui, repo, **kwargs): @@ -203,15 +243,20 @@ def post_pull_ssh(ui, repo, **kwargs): def pre_push(ui, repo, node=None, **kwargs): + """ + Mercurial pre_push hook + """ extras = _extras_from_ui(ui) + detect_force_push = extras.get('detect_force_push') rev_data = [] if node and kwargs.get('hooktype') == 'pretxnchangegroup': branches = collections.defaultdict(list) - for commit_id, branch in _rev_range_hash(repo, node): + commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push) + for commit_id, branch in commits: branches[branch].append(commit_id) - for branch, commits in branches.iteritems(): + for branch, commits in branches.items(): old_rev = kwargs.get('node_last') or commits[0] rev_data.append({ 'old_rev': old_rev, @@ -221,18 +266,25 @@ def pre_push(ui, repo, node=None, **kwar 'name': branch, }) + for push_ref in rev_data: + push_ref['multiple_heads'] = _heads + extras['commit_ids'] = rev_data return _call_hook('pre_push', extras, HgMessageWriter(ui)) def pre_push_ssh(ui, repo, node=None, **kwargs): - if _extras_from_ui(ui).get('SSH'): + extras = _extras_from_ui(ui) + if extras.get('SSH'): return pre_push(ui, repo, node, **kwargs) return 0 def pre_push_ssh_auth(ui, repo, node=None, **kwargs): + """ + Mercurial pre_push hook for SSH + """ extras = _extras_from_ui(ui) if extras.get('SSH'): permission = extras['SSH_PERMISSIONS'] @@ -247,6 +299,9 @@ def pre_push_ssh_auth(ui, repo, node=Non def post_push(ui, repo, node, **kwargs): + """ + Mercurial post_push hook + """ extras = _extras_from_ui(ui) commit_ids = [] @@ -254,7 +309,8 @@ def post_push(ui, repo, node, **kwargs): bookmarks = [] tags = [] - for commit_id, branch in _rev_range_hash(repo, node): + commits, _heads = _rev_range_hash(repo, node) + for commit_id, branch in commits: commit_ids.append(commit_id) if branch not in branches: branches.append(branch) @@ -273,6 +329,9 @@ def post_push(ui, repo, node, **kwargs): def post_push_ssh(ui, repo, node, **kwargs): + """ + Mercurial post_push hook for SSH + """ if _extras_from_ui(ui).get('SSH'): return post_push(ui, repo, node, **kwargs) return 0 @@ -389,6 +448,33 @@ def git_pre_receive(unused_repo_path, re rev_data = _parse_git_ref_lines(revision_lines) if 'push' not in extras['hooks']: return 0 + empty_commit_id = '0' * 40 + + detect_force_push = extras.get('detect_force_push') + + for push_ref in rev_data: + # store our git-env which holds the temp store + push_ref['git_env'] = [ + (k, v) for k, v in os.environ.items() if k.startswith('GIT')] + push_ref['pruned_sha'] = '' + if not detect_force_push: + # don't check for forced-push when we don't need to + continue + + type_ = push_ref['type'] + new_branch = push_ref['old_rev'] == empty_commit_id + if type_ == 'heads' and not new_branch: + old_rev = push_ref['old_rev'] + new_rev = push_ref['new_rev'] + cmd = [settings.GIT_EXECUTABLE, 'rev-list', + old_rev, '^{}'.format(new_rev)] + stdout, stderr = subprocessio.run_command( + cmd, env=os.environ.copy()) + # means we're having some non-reachable objects, this forced push + # was used + if stdout: + push_ref['pruned_sha'] = stdout.splitlines() + extras['commit_ids'] = rev_data return _call_hook('pre_push', extras, GitMessageWriter()) @@ -442,7 +528,8 @@ def git_post_receive(unused_repo_path, r cmd, env=os.environ.copy()) heads = stdout heads = heads.replace(push_ref['ref'], '') - heads = ' '.join(head for head in heads.splitlines() if head) + heads = ' '.join(head for head + in heads.splitlines() if head) or '.' cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse', '--pretty=format:%H', '--', push_ref['new_rev'], '--not', heads] diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -16,6 +16,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os +import sys import base64 import locale import logging @@ -26,17 +27,31 @@ from itertools import chain import simplejson as json import msgpack -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options from pyramid.config import Configurator +from pyramid.settings import asbool, aslist from pyramid.wsgi import wsgiapp from pyramid.compat import configparser + +log = logging.getLogger(__name__) + +# due to Mercurial/glibc2.27 problems we need to detect if locale settings are +# causing problems and "fix" it in case they do and fallback to LC_ALL = C + +try: + locale.setlocale(locale.LC_ALL, '') +except locale.Error as e: + log.error( + 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) + os.environ['LC_ALL'] = 'C' + + from vcsserver import remote_wsgi, scm_app, settings, hgpatches from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp -from vcsserver.exceptions import HTTPRepoLocked +from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected +from vcsserver.lib.exc_tracking import store_exception from vcsserver.server import VcsServer try: @@ -57,7 +72,7 @@ except ImportError: SubversionFactory = None SvnRemote = None -log = logging.getLogger(__name__) + def _is_request_chunked(environ): @@ -65,48 +80,60 @@ def _is_request_chunked(environ): return stream +def _int_setting(settings, name, default): + settings[name] = int(settings.get(name, default)) + + +def _bool_setting(settings, name, default): + input_val = settings.get(name, default) + if isinstance(input_val, unicode): + input_val = input_val.encode('utf8') + settings[name] = asbool(input_val) + + +def _list_setting(settings, name, default): + raw_value = settings.get(name, default) + + # Otherwise we assume it uses pyramids space/newline separation. + settings[name] = aslist(raw_value) + + +def _string_setting(settings, name, default, lower=True): + value = settings.get(name, default) + if lower: + value = value.lower() + settings[name] = value + + class VCS(object): def __init__(self, locale=None, cache_config=None): self.locale = locale self.cache_config = cache_config self._configure_locale() - self._initialize_cache() if GitFactory and GitRemote: - git_repo_cache = self.cache.get_cache_region( - 'git', region='repo_object') - git_factory = GitFactory(git_repo_cache) + git_factory = GitFactory() self._git_remote = GitRemote(git_factory) else: log.info("Git client import failed") if MercurialFactory and HgRemote: - hg_repo_cache = self.cache.get_cache_region( - 'hg', region='repo_object') - hg_factory = MercurialFactory(hg_repo_cache) + hg_factory = MercurialFactory() self._hg_remote = HgRemote(hg_factory) else: log.info("Mercurial client import failed") if SubversionFactory and SvnRemote: - svn_repo_cache = self.cache.get_cache_region( - 'svn', region='repo_object') - svn_factory = SubversionFactory(svn_repo_cache) + svn_factory = SubversionFactory() + # hg factory is used for svn url validation - hg_repo_cache = self.cache.get_cache_region( - 'hg', region='repo_object') - hg_factory = MercurialFactory(hg_repo_cache) + hg_factory = MercurialFactory() self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) else: log.info("Subversion client import failed") self._vcsserver = VcsServer() - def _initialize_cache(self): - cache_config = parse_cache_config_options(self.cache_config) - log.info('Initializing beaker cache: %s' % cache_config) - self.cache = CacheManager(**cache_config) - def _configure_locale(self): if self.locale: log.info('Settings locale: `LC_ALL` to %s' % self.locale) @@ -169,8 +196,11 @@ class HTTPApplication(object): _use_echo_app = False def __init__(self, settings=None, global_config=None): + self._sanitize_settings_and_apply_defaults(settings) + self.config = Configurator(settings=settings) self.global_config = global_config + self.config.include('vcsserver.lib.rc_cache') locale = settings.get('locale', '') or 'en_US.UTF-8' vcs = VCS(locale=locale, cache_config=settings) @@ -198,6 +228,21 @@ class HTTPApplication(object): if binary_dir: settings.BINARY_DIR = binary_dir + def _sanitize_settings_and_apply_defaults(self, settings): + # repo_object cache + _string_setting( + settings, + 'rc_cache.repo_object.backend', + 'dogpile.cache.rc.memory_lru') + _int_setting( + settings, + 'rc_cache.repo_object.expiration_time', + 300) + _int_setting( + settings, + 'rc_cache.repo_object.max_size', + 1024) + def _configure(self): self.config.add_renderer( name='msgpack', @@ -246,18 +291,35 @@ class HTTPApplication(object): wire = params.get('wire') args = params.get('args') kwargs = params.get('kwargs') + context_uid = None + if wire: try: - wire['context'] = uuid.UUID(wire['context']) + wire['context'] = context_uid = uuid.UUID(wire['context']) except KeyError: pass args.insert(0, wire) - log.debug('method called:%s with kwargs:%s', method, kwargs) + log.debug('method called:%s with kwargs:%s context_uid: %s', + method, kwargs, context_uid) try: resp = getattr(remote, method)(*args, **kwargs) except Exception as e: - tb_info = traceback.format_exc() + exc_info = list(sys.exc_info()) + exc_type, exc_value, exc_traceback = exc_info + + org_exc = getattr(e, '_org_exc', None) + org_exc_name = None + if org_exc: + org_exc_name = org_exc.__class__.__name__ + # replace our "faked" exception with our org + exc_info[0] = org_exc.__class__ + exc_info[1] = org_exc + + store_exception(id(exc_info), exc_info) + + tb_info = ''.join( + traceback.format_exception(exc_type, exc_value, exc_traceback)) type_ = e.__class__.__name__ if type_ not in self.ALLOWED_EXCEPTIONS: @@ -268,11 +330,12 @@ class HTTPApplication(object): 'error': { 'message': e.message, 'traceback': tb_info, + 'org_exc': org_exc_name, 'type': type_ } } try: - resp['error']['_vcs_kind'] = e._vcs_kind + resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None) except AttributeError: pass else: @@ -461,9 +524,21 @@ class HTTPApplication(object): return HTTPRepoLocked( title=exception.message, status_code=status_code) - # Re-raise exception if we can not handle it. - log.exception( - 'error occurred handling this request for path: %s', request.path) + elif _vcs_kind == 'repo_branch_protected': + # Get custom repo-branch-protected status code if present. + return HTTPRepoBranchProtected(title=exception.message) + + exc_info = request.exc_info + store_exception(id(exc_info), exc_info) + + traceback_info = 'unavailable' + if request.exc_info: + exc_type, exc_value, exc_tb = request.exc_info + traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb)) + + log.error( + 'error occurred handling this request for path: %s, \n tb: %s', + request.path, traceback_info) raise exception @@ -483,5 +558,6 @@ def main(global_config, **settings): if MercurialFactory: hgpatches.patch_largefiles_capabilities() hgpatches.patch_subrepo_type_mapping() + app = HTTPApplication(settings=settings, global_config=global_config) return app.wsgi_app() diff --git a/vcsserver/lib/__init__.py b/vcsserver/lib/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/__init__.py @@ -0,0 +1,16 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/vcsserver/lib/exc_tracking.py b/vcsserver/lib/exc_tracking.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/exc_tracking.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +import os +import time +import datetime +import msgpack +import logging +import traceback +import tempfile + + +log = logging.getLogger(__name__) + +# NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking +global_prefix = 'vcsserver' + + +def exc_serialize(exc_id, tb, exc_type): + + data = { + 'version': 'v1', + 'exc_id': exc_id, + 'exc_utc_date': datetime.datetime.utcnow().isoformat(), + 'exc_timestamp': repr(time.time()), + 'exc_message': tb, + 'exc_type': exc_type, + } + return msgpack.packb(data), data + + +def exc_unserialize(tb): + return msgpack.unpackb(tb) + + +def get_exc_store(): + """ + Get and create exception store if it's not existing + """ + exc_store_dir = 'rc_exception_store_v1' + # fallback + _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir) + + exc_store_dir = '' # TODO: need a persistent cross instance store here + if exc_store_dir: + _exc_store_path = os.path.join(exc_store_dir, exc_store_dir) + + _exc_store_path = os.path.abspath(_exc_store_path) + if not os.path.isdir(_exc_store_path): + os.makedirs(_exc_store_path) + log.debug('Initializing exceptions store at %s', _exc_store_path) + return _exc_store_path + + +def _store_exception(exc_id, exc_info, prefix): + exc_type, exc_value, exc_traceback = exc_info + tb = ''.join(traceback.format_exception( + exc_type, exc_value, exc_traceback, None)) + + exc_type_name = exc_type.__name__ + exc_store_path = get_exc_store() + exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name) + exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp']) + if not os.path.isdir(exc_store_path): + os.makedirs(exc_store_path) + stored_exc_path = os.path.join(exc_store_path, exc_pref_id) + with open(stored_exc_path, 'wb') as f: + f.write(exc_data) + log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path) + + +def store_exception(exc_id, exc_info, prefix=global_prefix): + try: + _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) + except Exception: + log.exception('Failed to store exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass + + +def _find_exc_file(exc_id, prefix=global_prefix): + exc_store_path = get_exc_store() + if prefix: + exc_id = '{}_{}'.format(exc_id, prefix) + else: + # search without a prefix + exc_id = '{}'.format(exc_id) + + # we need to search the store for such start pattern as above + for fname in os.listdir(exc_store_path): + if fname.startswith(exc_id): + exc_id = os.path.join(exc_store_path, fname) + break + continue + else: + exc_id = None + + return exc_id + + +def _read_exception(exc_id, prefix): + exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix) + if exc_id_file_path: + with open(exc_id_file_path, 'rb') as f: + return exc_unserialize(f.read()) + else: + log.debug('Exception File `%s` not found', exc_id_file_path) + return None + + +def read_exception(exc_id, prefix=global_prefix): + try: + return _read_exception(exc_id=exc_id, prefix=prefix) + except Exception: + log.exception('Failed to read exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + return None + + +def delete_exception(exc_id, prefix=global_prefix): + try: + exc_id_file_path = _find_exc_file(exc_id, prefix=prefix) + if exc_id_file_path: + os.remove(exc_id_file_path) + + except Exception: + log.exception('Failed to remove exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass diff --git a/vcsserver/lib/memory_lru_dict.py b/vcsserver/lib/memory_lru_dict.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/memory_lru_dict.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +import logging + +from repoze.lru import LRUCache + +from vcsserver.utils import safe_str + +log = logging.getLogger(__name__) + + +class LRUDict(LRUCache): + """ + Wrapper to provide partial dict access + """ + + def __setitem__(self, key, value): + return self.put(key, value) + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, key): + return bool(self.get(key)) + + def __delitem__(self, key): + del self.data[key] + + def keys(self): + return self.data.keys() + + +class LRUDictDebug(LRUDict): + """ + Wrapper to provide some debug options + """ + def _report_keys(self): + elems_cnt = '%s/%s' % (len(self.keys()), self.size) + # trick for pformat print it more nicely + fmt = '\n' + for cnt, elem in enumerate(self.keys()): + fmt += '%s - %s\n' % (cnt+1, safe_str(elem)) + log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt)) + + def __getitem__(self, key): + self._report_keys() + return self.get(key) diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/__init__.py @@ -0,0 +1,60 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging +from dogpile.cache import register_backend + +register_backend( + "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends", + "LRUMemoryBackend") + +log = logging.getLogger(__name__) + +from . import region_meta +from .util import key_generator, get_default_cache_settings, make_region + + +def configure_dogpile_cache(settings): + cache_dir = settings.get('cache_dir') + if cache_dir: + region_meta.dogpile_config_defaults['cache_dir'] = cache_dir + + rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.']) + + # inspect available namespaces + avail_regions = set() + for key in rc_cache_data.keys(): + namespace_name = key.split('.', 1)[0] + avail_regions.add(namespace_name) + log.debug('dogpile: found following cache regions: %s', avail_regions) + + # register them into namespace + for region_name in avail_regions: + new_region = make_region( + name=region_name, + function_key_generator=key_generator + ) + + new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) + + log.debug('dogpile: registering a new region %s[%s]', + region_name, new_region.__dict__) + region_meta.dogpile_cache_regions[region_name] = new_region + + +def includeme(config): + configure_dogpile_cache(config.registry.settings) diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/backends.py @@ -0,0 +1,51 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging + +from dogpile.cache.backends import memory as memory_backend +from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug + + +_default_max_size = 1024 + +log = logging.getLogger(__name__) + + +class LRUMemoryBackend(memory_backend.MemoryBackend): + pickle_values = False + + def __init__(self, arguments): + max_size = arguments.pop('max_size', _default_max_size) + + LRUDictClass = LRUDict + if arguments.pop('log_key_count', None): + LRUDictClass = LRUDictDebug + + arguments['cache_dict'] = LRUDictClass(max_size) + super(LRUMemoryBackend, self).__init__(arguments) + + def delete(self, key): + try: + del self._cache[key] + except KeyError: + # we don't care if key isn't there at deletion + pass + + def delete_multi(self, keys): + for key in keys: + self.delete(key) diff --git a/vcsserver/lib/rc_cache/region_meta.py b/vcsserver/lib/rc_cache/region_meta.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/region_meta.py @@ -0,0 +1,26 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import tempfile + +dogpile_config_defaults = { + 'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache') +} + +# GLOBAL TO STORE ALL REGISTERED REGIONS +dogpile_cache_regions = {} diff --git a/vcsserver/lib/rc_cache/util.py b/vcsserver/lib/rc_cache/util.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/rc_cache/util.py @@ -0,0 +1,136 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import logging +import functools + +from vcsserver.utils import safe_str, sha1 +from dogpile.cache import CacheRegion +from dogpile.cache.util import compat + +log = logging.getLogger(__name__) + + +class RhodeCodeCacheRegion(CacheRegion): + + def conditional_cache_on_arguments( + self, namespace=None, + expiration_time=None, + should_cache_fn=None, + to_str=compat.string_type, + function_key_generator=None, + condition=True): + """ + Custom conditional decorator, that will not touch any dogpile internals if + condition isn't meet. This works a bit different than should_cache_fn + And it's faster in cases we don't ever want to compute cached values + """ + expiration_time_is_callable = compat.callable(expiration_time) + + if function_key_generator is None: + function_key_generator = self.function_key_generator + + def decorator(fn): + if to_str is compat.string_type: + # backwards compatible + key_generator = function_key_generator(namespace, fn) + else: + key_generator = function_key_generator(namespace, fn, to_str=to_str) + + @functools.wraps(fn) + def decorate(*arg, **kw): + key = key_generator(*arg, **kw) + + @functools.wraps(fn) + def creator(): + return fn(*arg, **kw) + + if not condition: + return creator() + + timeout = expiration_time() if expiration_time_is_callable \ + else expiration_time + + return self.get_or_create(key, creator, timeout, should_cache_fn) + + def invalidate(*arg, **kw): + key = key_generator(*arg, **kw) + self.delete(key) + + def set_(value, *arg, **kw): + key = key_generator(*arg, **kw) + self.set(key, value) + + def get(*arg, **kw): + key = key_generator(*arg, **kw) + return self.get(key) + + def refresh(*arg, **kw): + key = key_generator(*arg, **kw) + value = fn(*arg, **kw) + self.set(key, value) + return value + + decorate.set = set_ + decorate.invalidate = invalidate + decorate.refresh = refresh + decorate.get = get + decorate.original = fn + decorate.key_generator = key_generator + + return decorate + + return decorator + + +def make_region(*arg, **kw): + return RhodeCodeCacheRegion(*arg, **kw) + + +def get_default_cache_settings(settings, prefixes=None): + prefixes = prefixes or [] + cache_settings = {} + for key in settings.keys(): + for prefix in prefixes: + if key.startswith(prefix): + name = key.split(prefix)[1].strip() + val = settings[key] + if isinstance(val, basestring): + val = val.strip() + cache_settings[name] = val + return cache_settings + + +def compute_key_from_params(*args): + """ + Helper to compute key from given params to be used in cache manager + """ + return sha1("_".join(map(safe_str, args))) + + +def key_generator(namespace, fn): + fname = fn.__name__ + + def generate_key(*args): + namespace_pref = namespace or 'default' + arg_key = compute_key_from_params(*args) + final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) + + return final_key + + return generate_key diff --git a/vcsserver/scm_app.py b/vcsserver/scm_app.py --- a/vcsserver/scm_app.py +++ b/vcsserver/scm_app.py @@ -21,9 +21,9 @@ import itertools import mercurial import mercurial.error +import mercurial.wireprotoserver import mercurial.hgweb.common import mercurial.hgweb.hgweb_mod -import mercurial.hgweb.protocol import webob.exc from vcsserver import pygrack, exceptions, settings, git_lfs @@ -73,14 +73,18 @@ class HgWeb(mercurial.hgweb.hgweb_mod.hg This may be called by multiple threads. """ - req = mercurial.hgweb.request.wsgirequest(environ, start_response) - gen = self.run_wsgi(req) + from mercurial.hgweb import request as requestmod + req = requestmod.parserequestfromenv(environ) + res = requestmod.wsgiresponse(req, start_response) + gen = self.run_wsgi(req, res) first_chunk = None try: data = gen.next() - def first_chunk(): yield data + + def first_chunk(): + yield data except StopIteration: pass @@ -88,17 +92,18 @@ class HgWeb(mercurial.hgweb.hgweb_mod.hg return itertools.chain(first_chunk(), gen) return gen - def _runwsgi(self, req, repo): - cmd = req.form.get('cmd', [''])[0] - if not mercurial.hgweb.protocol.iscmd(cmd): - req.respond( - mercurial.hgweb.common.ErrorResponse( - mercurial.hgweb.common.HTTP_BAD_REQUEST), - mercurial.hgweb.protocol.HGTYPE - ) - return [''] + def _runwsgi(self, req, res, repo): - return super(HgWeb, self)._runwsgi(req, repo) + cmd = req.qsparams.get('cmd', '') + if not mercurial.wireprotoserver.iscmd(cmd): + # NOTE(marcink): for unsupported commands, we return bad request + # internally from HG + from mercurial.hgweb.common import statusmessage + res.status = statusmessage(mercurial.hgweb.common.HTTP_BAD_REQUEST) + res.setbodybytes('') + return res.sendresponse() + + return super(HgWeb, self)._runwsgi(req, res, repo) def make_hg_ui_from_config(repo_config): @@ -147,8 +152,8 @@ def create_hg_wsgi_app(repo_path, repo_n try: return HgWeb(repo_path, name=repo_name, baseui=baseui) - except mercurial.error.RequirementError as exc: - raise exceptions.RequirementException(exc) + except mercurial.error.RequirementError as e: + raise exceptions.RequirementException(e)(e) class GitHandler(object): diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -40,13 +40,13 @@ log = logging.getLogger(__name__) # Set of svn compatible version flags. # Compare with subversion/svnadmin/svnadmin.c -svn_compatible_versions = set([ +svn_compatible_versions = { 'pre-1.4-compatible', 'pre-1.5-compatible', 'pre-1.6-compatible', 'pre-1.8-compatible', - 'pre-1.9-compatible', -]) + 'pre-1.9-compatible' +} svn_compatible_versions_map = { 'pre-1.4-compatible': '1.3', @@ -64,13 +64,14 @@ def reraise_safe_exceptions(func): return func(*args, **kwargs) except Exception as e: if not hasattr(e, '_vcs_kind'): - log.exception("Unhandled exception in hg remote call") - raise_from_original(exceptions.UnhandledException) + log.exception("Unhandled exception in svn remote call") + raise_from_original(exceptions.UnhandledException(e)) raise return wrapper class SubversionFactory(RepoFactory): + repo_type = 'svn' def _create_repo(self, wire, create, compatible_version): path = svn.core.svn_path_canonicalize(wire['path']) @@ -92,10 +93,25 @@ class SubversionFactory(RepoFactory): return repo def repo(self, wire, create=False, compatible_version=None): - def create_new_repo(): + """ + Get a repository instance for the given path. + + Uses internally the low level beaker API since the decorators introduce + significant overhead. + """ + region = self._cache_region + context = wire.get('context', None) + repo_path = wire.get('path', '') + context_uid = '{}'.format(context) + cache = wire.get('cache', True) + cache_on = context and cache + + @region.conditional_cache_on_arguments(condition=cache_on) + def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id): return self._create_repo(wire, create, compatible_version) - return self._repo(wire, create_new_repo) + return create_new_repo(self.repo_type, repo_path, context_uid, + compatible_version) NODE_TYPE_MAPPING = { diff --git a/vcsserver/tests/conftest.py b/vcsserver/tests/conftest.py --- a/vcsserver/tests/conftest.py +++ b/vcsserver/tests/conftest.py @@ -40,7 +40,7 @@ def repeat(request): @pytest.fixture(scope='session') def vcsserver_port(request): port = get_available_port() - print 'Using vcsserver port %s' % (port, ) + print('Using vcsserver port %s' % (port, )) return port @@ -55,4 +55,3 @@ def get_available_port(): mysocket.close() del mysocket return port - diff --git a/vcsserver/tests/test_git.py b/vcsserver/tests/test_git.py --- a/vcsserver/tests/test_git.py +++ b/vcsserver/tests/test_git.py @@ -152,11 +152,14 @@ class TestDulwichRepoWrapper(object): class TestGitFactory(object): def test_create_repo_returns_dulwich_wrapper(self): - factory = git.GitFactory(repo_cache=Mock()) - wire = { - 'path': '/tmp/abcde' - } - isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) - with isdir_patcher: - result = factory._create_repo(wire, True) - assert isinstance(result, git.Repo) + + with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock: + mock.side_effect = {'repo_objects': ''} + factory = git.GitFactory() + wire = { + 'path': '/tmp/abcde' + } + isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) + with isdir_patcher: + result = factory._create_repo(wire, True) + assert isinstance(result, git.Repo) diff --git a/vcsserver/tests/test_hg.py b/vcsserver/tests/test_hg.py --- a/vcsserver/tests/test_hg.py +++ b/vcsserver/tests/test_hg.py @@ -120,7 +120,7 @@ class TestReraiseSafeExceptions(object): def test_does_not_map_known_exceptions(self): @hg.reraise_safe_exceptions def stub_method(): - raise exceptions.LookupException('stub') + raise exceptions.LookupException()('stub') with pytest.raises(Exception) as exc_info: stub_method() diff --git a/vcsserver/tests/test_hgpatches.py b/vcsserver/tests/test_hgpatches.py --- a/vcsserver/tests/test_hgpatches.py +++ b/vcsserver/tests/test_hgpatches.py @@ -28,56 +28,45 @@ def test_patch_largefiles_capabilities_a patched_capabilities): lfproto = hgcompat.largefiles.proto hgpatches.patch_largefiles_capabilities() - assert lfproto.capabilities.func_name == '_dynamic_capabilities' + assert lfproto._capabilities.func_name == '_dynamic_capabilities' def test_dynamic_capabilities_uses_original_function_if_not_enabled( - stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities): + stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities, + orig_capabilities): dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper( hgcompat.largefiles.proto, stub_extensions) - caps = dynamic_capabilities(stub_repo, stub_proto) + caps = dynamic_capabilities(orig_capabilities, stub_repo, stub_proto) stub_extensions.assert_called_once_with(stub_ui) assert LARGEFILES_CAPABILITY not in caps -def test_dynamic_capabilities_uses_updated_capabilitiesorig( - stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities): - dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper( - hgcompat.largefiles.proto, stub_extensions) - - # This happens when the extension is loaded for the first time, important - # to ensure that an updated function is correctly picked up. - hgcompat.largefiles.proto.capabilitiesorig = mock.Mock( - return_value='REPLACED') - - caps = dynamic_capabilities(stub_repo, stub_proto) - assert 'REPLACED' == caps - - def test_dynamic_capabilities_ignores_updated_capabilities( - stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities): + stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities, + orig_capabilities): stub_extensions.return_value = [('largefiles', mock.Mock())] dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper( hgcompat.largefiles.proto, stub_extensions) # This happens when the extension is loaded for the first time, important # to ensure that an updated function is correctly picked up. - hgcompat.largefiles.proto.capabilities = mock.Mock( + hgcompat.largefiles.proto._capabilities = mock.Mock( side_effect=Exception('Must not be called')) - dynamic_capabilities(stub_repo, stub_proto) + dynamic_capabilities(orig_capabilities, stub_repo, stub_proto) def test_dynamic_capabilities_uses_largefiles_if_enabled( - stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities): + stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities, + orig_capabilities): stub_extensions.return_value = [('largefiles', mock.Mock())] dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper( hgcompat.largefiles.proto, stub_extensions) - caps = dynamic_capabilities(stub_repo, stub_proto) + caps = dynamic_capabilities(orig_capabilities, stub_repo, stub_proto) stub_extensions.assert_called_once_with(stub_ui) assert LARGEFILES_CAPABILITY in caps @@ -94,15 +83,11 @@ def patched_capabilities(request): Patch in `capabilitiesorig` and restore both capability functions. """ lfproto = hgcompat.largefiles.proto - orig_capabilities = lfproto.capabilities - orig_capabilitiesorig = lfproto.capabilitiesorig - - lfproto.capabilitiesorig = mock.Mock(return_value='ORIG') + orig_capabilities = lfproto._capabilities @request.addfinalizer def restore(): - lfproto.capabilities = orig_capabilities - lfproto.capabilitiesorig = orig_capabilitiesorig + lfproto._capabilities = orig_capabilities @pytest.fixture @@ -120,6 +105,15 @@ def stub_proto(stub_ui): @pytest.fixture +def orig_capabilities(): + from mercurial.wireprotov1server import wireprotocaps + + def _capabilities(repo, proto): + return wireprotocaps + return _capabilities + + +@pytest.fixture def stub_ui(): return hgcompat.ui.ui() diff --git a/vcsserver/tests/test_http_performance.py b/vcsserver/tests/test_http_performance.py --- a/vcsserver/tests/test_http_performance.py +++ b/vcsserver/tests/test_http_performance.py @@ -12,11 +12,6 @@ from vcsserver.http_main import main def vcs_app(): stub_settings = { 'dev.use_echo_app': 'true', - 'beaker.cache.regions': 'repo_object', - 'beaker.cache.repo_object.type': 'memorylru', - 'beaker.cache.repo_object.max_items': '100', - 'beaker.cache.repo_object.expire': '300', - 'beaker.cache.repo_object.enabled': 'true', 'locale': 'en_US.UTF-8', } vcs_app = main({}, **stub_settings) diff --git a/vcsserver/utils.py b/vcsserver/utils.py --- a/vcsserver/utils.py +++ b/vcsserver/utils.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging +import hashlib log = logging.getLogger(__name__) @@ -80,3 +81,9 @@ class AttributeDict(dict): return self.get(attr, None) __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ + + +def sha1(val): + return hashlib.sha1(val).hexdigest() + +