release: Merge default into stable for release preparation

requirements_debug.txt

@@ -0,0 +1,8 b''
+## special libraries we could extend the requirements.txt file with to add some
+## custom libraries useful for debug and memory tracing
+
+## uncomment inclusion of this file in requirements.txt run make generate-pkgs and nix-shell
+
+objgraph
+memory-profiler
+pympler

vcsserver/lib/request_counter.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2019 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+counter = 0
+
+
+def get_request_counter(request):
+    global counter
+    counter += 1
+    return counter

vcsserver/tweens/__init__.py

@@ -0,0 +1,19 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

vcsserver/vcs_base.py

@@ -0,0 +1,32 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2019 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+class RemoteBase(object):
+    EMPTY_COMMIT = '0' * 40
+
+    @property
+    def region(self):
+        return self._factory._cache_region
+
+    def _cache_on(self, wire):
+        context = wire.get('context', '')
+        context_uid = '{}'.format(context)
+        repo_id = wire.get('repo_id', '')
+        cache = wire.get('cache', True)
+        cache_on = context and cache
+        return cache_on, context_uid, repo_id

.bumpversion.cfg

@@ -1,6 +1,6 b''
 [bumpversion]
-current_version = 4.17.4
+current_version = 4.18.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]
 

.release.cfg

@@ -1,16 +1,14 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.17.4
+state = in_progress
+version = 4.18.0
 

configs/development.ini

@@ -1,87 +1,237 b''
-################################################################################
-# RhodeCode VCSServer with HTTP Backend - configuration                        #
-################################################################################
+## -*- coding: utf-8 -*-
 
+; #################################
+; RHODECODE VCSSERVER CONFIGURATION
+; #################################
 
 [server:main]
-## COMMON ##
+; COMMON HOST/IP CONFIG
 host = 0.0.0.0
 port = 9900
 
+; ##################################################
+; WAITRESS WSGI SERVER - Recommended for Development
+; ##################################################
+
+; use server type
 use = egg:waitress#main
 
+; number of worker threads
+threads = 5
+
+; MAX BODY SIZE 100GB
+max_request_body_size = 107374182400
+
+; Use poll instead of select, fixes file descriptors limits problems.
+; May not work on old windows systems.
+asyncore_use_poll = true
+
+
+; ###########################
+; GUNICORN APPLICATION SERVER
+; ###########################
+
+; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; Module to use, this setting shouldn't be changed
+#use = egg:gunicorn#main
+
+; Sets the number of process workers. More workers means more concurrent connections
+; RhodeCode can handle at the same time. Each additional worker also it increases
+; memory usage as each has it's own set of caches.
+; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+; than 8-10 unless for really big deployments .e.g 700-1000 users.
+; `instance_id = *` must be set in the [app:main] section below (which is the default)
+; when using more than 1 worker.
+#workers = 2
+
+; Gunicorn access log level
+#loglevel = info
+
+; Process name visible in process list
+#proc_name = rhodecode_vcsserver
+
+; Type of worker class, one of `sync`, `gevent`
+; currently `sync` is the only option allowed.
+#worker_class = sync
+
+; The maximum number of simultaneous clients. Valid only for gevent
+#worker_connections = 10
+
+; Max number of requests that worker will handle before being gracefully restarted.
+; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
+#max_requests = 1000
+#max_requests_jitter = 30
+
+; Amount of time a worker can spend with handling a request before it
+; gets killed and restarted. By default set to 21600 (6hrs)
+; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+#timeout = 21600
+
+; The maximum size of HTTP request line in bytes.
+; 0 for unlimited
+#limit_request_line = 0
+
+; Limit the number of HTTP headers fields in a request.
+; By default this value is 100 and can't be larger than 32768.
+#limit_request_fields = 32768
+
+; Limit the allowed size of an HTTP request header field.
+; Value is a positive number or 0.
+; Setting it to 0 will allow unlimited header field sizes.
+#limit_request_field_size = 0
+
+; Timeout for graceful workers restart.
+; After receiving a restart signal, workers have this much time to finish
+; serving requests. Workers still alive after the timeout (starting from the
+; receipt of the restart signal) are force killed.
+; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+#graceful_timeout = 3600
+
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range.
+#keepalive = 2
+
+; Maximum memory usage that each worker can use before it will receive a
+; graceful restart signal 0 = memory monitoring is disabled
+; Examples: 268435456 (256MB), 536870912 (512MB)
+; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+#memory_max_usage = 0
+
+; How often in seconds to check for memory usage for each gunicorn worker
+#memory_usage_check_interval = 60
+
+; Threshold value for which we don't recycle worker if GarbageCollection
+; frees up enough resources. Before each restart we try to run GC on worker
+; in case we get enough free memory after that, restart will not happen.
+#memory_usage_recovery_threshold = 0.8
+
 
 [app:main]
+; The %(here)s variable will be replaced with the absolute path of parent directory
+; of this file
 use = egg:rhodecode-vcsserver
 
-pyramid.default_locale_name = en
+
+; #############
+; DEBUG OPTIONS
+; #############
+
+# During development the we want to have the debug toolbar enabled
 pyramid.includes =
+    pyramid_debugtoolbar
 
-## default locale used by VCS systems
+debugtoolbar.hosts = 0.0.0.0/0
+debugtoolbar.exclude_prefixes =
+    /css
+    /fonts
+    /images
+    /js
+
+; #################
+; END DEBUG OPTIONS
+; #################
+
+; Pyramid default locales, we need this to be set
+pyramid.default_locale_name = en
+
+; default locale used by VCS systems
 locale = en_US.UTF-8
 
-
-## path to binaries for vcsserver, it should be set by the installer
-## at installation time, e.g /home/user/vcsserver-1/profile/bin
+; path to binaries for vcsserver, it should be set by the installer
+; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
-## Custom exception store path, defaults to TMPDIR
-## This is used to store exception from RhodeCode in shared directory
+; Custom exception store path, defaults to TMPDIR
+; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
-## Default cache dir for caches.  Putting this into a ramdisk
-## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
-## large amount of space
-cache_dir = %(here)s/rcdev/data
+; #############
+; DOGPILE CACHE
+; #############
+
+; Default cache dir for caches. Putting this into a ramdisk can boost performance.
+; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
+cache_dir = %(here)s/data
+
+; ***************************************
+; `repo_object` cache, default file based
+; ***************************************
+
+; `repo_object` cache settings for vcs methods for repositories
+rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+
+; cache auto-expires after N seconds
+; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+rc_cache.repo_object.expiration_time = 2592000
+
+; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
 
-## cache region for storing repo_objects cache
-rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
-## cache auto-expires after N seconds
-rc_cache.repo_object.expiration_time = 300
-## max size of LRU, old values will be discarded if the size of cache reaches max_size
-rc_cache.repo_object.max_size = 100
+; ***********************************************************
+; `repo_object` cache with redis backend
+; recommended for larger instance, and for better performance
+; ***********************************************************
+
+; `repo_object` cache settings for vcs methods for repositories
+#rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
+
+; cache auto-expires after N seconds
+; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+#rc_cache.repo_object.expiration_time = 2592000
+
+; redis_expiration_time needs to be greater then expiration_time
+#rc_cache.repo_object.arguments.redis_expiration_time = 3592000
+
+#rc_cache.repo_object.arguments.host = localhost
+#rc_cache.repo_object.arguments.port = 6379
+#rc_cache.repo_object.arguments.db = 5
+#rc_cache.repo_object.arguments.socket_timeout = 30
+; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
+#rc_cache.repo_object.arguments.distributed_lock = true
 
 
-################################
-### LOGGING CONFIGURATION   ####
-################################
+; #####################
+; LOGGING CONFIGURATION
+; #####################
 [loggers]
 keys = root, vcsserver
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic
 
-#############
-## LOGGERS ##
-#############
+; #######
+; LOGGERS
+; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_vcsserver]
 level = DEBUG
 handlers =
 qualname = vcsserver
 propagate = 1
 
 
-##############
-## HANDLERS ##
-##############
+; ########
+; HANDLERS
+; ########
 
 [handler_console]
 class = StreamHandler
-args = (sys.stderr,)
+args = (sys.stderr, )
 level = DEBUG
 formatter = generic
 
-################
-## FORMATTERS ##
-################
+; ##########
+; FORMATTERS
+; ##########
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

configs/gunicorn_config.py

@@ -1,154 +1,265 b''
 """
-gunicorn config extension and hooks. Sets additional configuration that is
-available post the .ini config.
-
-- workers = ${cpu_number}
-- threads = 1
-- proc_name = ${gunicorn_proc_name}
-- worker_class = sync
-- worker_connections = 10
-- max_requests = 1000
-- max_requests_jitter = 30
-- timeout = 21600
-
+Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
+Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
-import multiprocessing
+import gc
+import os
 import sys
+import math
 import time
-import datetime
 import threading
 import traceback
+import random
 from gunicorn.glogging import Logger
 
 
+def get_workers():
+    import multiprocessing
+    return multiprocessing.cpu_count() * 2 + 1
+
 # GLOBAL
 errorlog = '-'
 accesslog = '-'
-loglevel = 'debug'
-
-# SECURITY
-
-# The maximum size of HTTP request line in bytes.
-# 0 for unlimited
-limit_request_line = 0
-
-# Limit the number of HTTP headers fields in a request.
-# By default this value is 100 and can't be larger than 32768.
-limit_request_fields = 10240
-
-# Limit the allowed size of an HTTP request header field.
-# Value is a positive number or 0.
-# Setting it to 0 will allow unlimited header field sizes.
-limit_request_field_size = 0
-
-
-# Timeout for graceful workers restart.
-# After receiving a restart signal, workers have this much time to finish
-# serving requests. Workers still alive after the timeout (starting from the
-# receipt of the restart signal) are force killed.
-graceful_timeout = 30
-
-
-# The number of seconds to wait for requests on a Keep-Alive connection.
-# Generally set in the 1-5 seconds range.
-keepalive = 2
 
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # Custom log format
 access_log_format = (
-    '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # self adjust workers based on CPU count
-# workers = multiprocessing.cpu_count() * 2 + 1
+# workers = get_workers()
+
+
+def _get_process_rss(pid=None):
+    try:
+        import psutil
+        if pid:
+            proc = psutil.Process(pid)
+        else:
+            proc = psutil.Process()
+        return proc.memory_info().rss
+    except Exception:
+        return None
 
 
-def post_fork(server, worker):
-    server.log.info("[<%-10s>] WORKER spawned", worker.pid)
+def _get_config(ini_path):
+
+    try:
+        import configparser
+    except ImportError:
+        import ConfigParser as configparser
+    try:
+        config = configparser.RawConfigParser()
+        config.read(ini_path)
+        return config
+    except Exception:
+        return None
+
+
+def _time_with_offset(memory_usage_check_interval):
+    return time.time() - random.randint(0, memory_usage_check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
+def post_fork(server, worker):
+
+    # memory spec defaults
+    _memory_max_usage = 0
+    _memory_usage_check_interval = 60
+    _memory_usage_recovery_threshold = 0.8
+
+    ini_path = os.path.abspath(server.cfg.paste)
+    conf = _get_config(ini_path)
+
+    section = 'server:main'
+    if conf and conf.has_section(section):
+
+        if conf.has_option(section, 'memory_max_usage'):
+            _memory_max_usage = conf.getint(section, 'memory_max_usage')
+
+        if conf.has_option(section, 'memory_usage_check_interval'):
+            _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
+
+        if conf.has_option(section, 'memory_usage_recovery_threshold'):
+            _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
+
+    worker._memory_max_usage = _memory_max_usage
+    worker._memory_usage_check_interval = _memory_usage_check_interval
+    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+
+    # register memory last check time, with some random offset so we don't recycle all
+    # at once
+    worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
+
+    if _memory_max_usage:
+        server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
+                        _format_data_size(_memory_max_usage))
+    else:
+        server.log.info("[%-10s] WORKER spawned", worker.pid)
+
+
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
-    server.log.info("Server is starting.")
+    server_lbl = '{} {}'.format(server.proc_name, server.address)
+    server.log.info("Server %s is starting.", server_lbl)
 
 
 def when_ready(server):
-    server.log.info("Server is ready. Spawning workers")
+    server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
+def _format_data_size(size, unit="B", precision=1, binary=True):
+    """Format a number using SI units (kilo, mega, etc.).
+
+    ``size``: The number as a float or int.
+
+    ``unit``: The unit name in plural form. Examples: "bytes", "B".
+
+    ``precision``: How many digits to the right of the decimal point. Default
+    is 1.  0 suppresses the decimal point.
+
+    ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
+    If true, use base-2 binary prefixes (kibi = Ki = 1024).
+
+    ``full_name``: If false (default), use the prefix abbreviation ("k" or
+    "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
+    use abbreviation ("k" or "Ki").
+
+    """
+
+    if not binary:
+        base = 1000
+        multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
+    else:
+        base = 1024
+        multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
+
+    sign = ""
+    if size > 0:
+        m = int(math.log(size, base))
+    elif size < 0:
+        sign = "-"
+        size = -size
+        m = int(math.log(size, base))
+    else:
+        m = 0
+    if m > 8:
+        m = 8
+
+    if m == 0:
+        precision = '%.0f'
+    else:
+        precision = '%%.%df' % precision
+
+    size = precision % (size / math.pow(base, m))
+
+    return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
+
+
+def _check_memory_usage(worker):
+    memory_max_usage = worker._memory_max_usage
+    if not memory_max_usage:
+        return
+
+    memory_usage_check_interval = worker._memory_usage_check_interval
+    memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
+
+    elapsed = time.time() - worker._last_memory_check_time
+    if elapsed > memory_usage_check_interval:
+        mem_usage = _get_process_rss()
+        if mem_usage and mem_usage > memory_max_usage:
+            worker.log.info(
+                "memory usage %s > %s, forcing gc",
+                _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+            # Try to clean it up by forcing a full collection.
+            gc.collect()
+            mem_usage = _get_process_rss()
+            if mem_usage > memory_usage_recovery_threshold:
+                # Didn't clean up enough, we'll have to terminate.
+                worker.log.warning(
+                    "memory usage %s > %s after gc, quitting",
+                    _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+                # This will cause worker to auto-restart itself
+                worker.alive = False
+        worker._last_memory_check_time = time.time()
+
+
 def worker_int(worker):
-    worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
+    worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
     code = []
     for thread_id, stack in sys._current_frames().items():
         code.append(
             "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
-    worker.log.info("[<%-10s>] worker received SIGABRT signal", worker.pid)
+    worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
-    worker.log.info("[<%-10s>] worker exit", worker.pid)
+    worker.log.info("[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
-    worker.log.info("[<%-10s>] worker child exit", worker.pid)
+    worker.log.info("[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
+    # Gunicorn sometimes has problems with reading the status_code
+    status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
-        "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.3fs",
-        worker.nr, req.method, req.path, resp.status_code, total_time)
+        "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
+        worker.nr, req.method, req.path, status_code, total_time)
+    _check_memory_usage(worker)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - long(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,108 +1,200 b''
-################################################################################
-# RhodeCode VCSServer with HTTP Backend - configuration                        #
-################################################################################
+## -*- coding: utf-8 -*-
 
+; #################################
+; RHODECODE VCSSERVER CONFIGURATION
+; #################################
 
 [server:main]
-## COMMON ##
+; COMMON HOST/IP CONFIG
 host = 127.0.0.1
 port = 9900
 
 
-##########################
-## GUNICORN WSGI SERVER ##
-##########################
-## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
+; ###########################
+; GUNICORN APPLICATION SERVER
+; ###########################
+
+; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
-## Sets the number of process workers. Recommended
-## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
+
+; Sets the number of process workers. More workers means more concurrent connections
+; RhodeCode can handle at the same time. Each additional worker also it increases
+; memory usage as each has it's own set of caches.
+; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+; than 8-10 unless for really big deployments .e.g 700-1000 users.
+; `instance_id = *` must be set in the [app:main] section below (which is the default)
+; when using more than 1 worker.
 workers = 2
-## process name
+
+; Gunicorn access log level
+loglevel = info
+
+; Process name visible in process list
 proc_name = rhodecode_vcsserver
-## type of worker class, currently `sync` is the only option allowed.
+
+; Type of worker class, one of `sync`, `gevent`
+; currently `sync` is the only option allowed.
 worker_class = sync
-## The maximum number of simultaneous clients. Valid only for Gevent
-#worker_connections = 10
-## max number of requests that worker will handle before being gracefully
-## restarted, could prevent memory leaks
+
+; The maximum number of simultaneous clients. Valid only for gevent
+worker_connections = 10
+
+; Max number of requests that worker will handle before being gracefully restarted.
+; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 max_requests = 1000
 max_requests_jitter = 30
-## amount of time a worker can spend with handling a request before it
-## gets killed and restarted. Set to 6hrs
+
+; Amount of time a worker can spend with handling a request before it
+; gets killed and restarted. By default set to 21600 (6hrs)
+; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 timeout = 21600
 
+; The maximum size of HTTP request line in bytes.
+; 0 for unlimited
+limit_request_line = 0
+
+; Limit the number of HTTP headers fields in a request.
+; By default this value is 100 and can't be larger than 32768.
+limit_request_fields = 32768
+
+; Limit the allowed size of an HTTP request header field.
+; Value is a positive number or 0.
+; Setting it to 0 will allow unlimited header field sizes.
+limit_request_field_size = 0
+
+; Timeout for graceful workers restart.
+; After receiving a restart signal, workers have this much time to finish
+; serving requests. Workers still alive after the timeout (starting from the
+; receipt of the restart signal) are force killed.
+; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+graceful_timeout = 3600
+
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range.
+keepalive = 2
+
+; Maximum memory usage that each worker can use before it will receive a
+; graceful restart signal 0 = memory monitoring is disabled
+; Examples: 268435456 (256MB), 536870912 (512MB)
+; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+memory_max_usage = 0
+
+; How often in seconds to check for memory usage for each gunicorn worker
+memory_usage_check_interval = 60
+
+; Threshold value for which we don't recycle worker if GarbageCollection
+; frees up enough resources. Before each restart we try to run GC on worker
+; in case we get enough free memory after that, restart will not happen.
+memory_usage_recovery_threshold = 0.8
+
 
 [app:main]
+; The %(here)s variable will be replaced with the absolute path of parent directory
+; of this file
 use = egg:rhodecode-vcsserver
 
+; Pyramid default locales, we need this to be set
 pyramid.default_locale_name = en
-pyramid.includes =
 
-## default locale used by VCS systems
+; default locale used by VCS systems
 locale = en_US.UTF-8
 
-
-## path to binaries for vcsserver, it should be set by the installer
-## at installation time, e.g /home/user/vcsserver-1/profile/bin
+; path to binaries for vcsserver, it should be set by the installer
+; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
-## Custom exception store path, defaults to TMPDIR
-## This is used to store exception from RhodeCode in shared directory
+; Custom exception store path, defaults to TMPDIR
+; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
-## Default cache dir for caches.  Putting this into a ramdisk
-## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
-## large amount of space
-cache_dir = %(here)s/rcdev/data
+; #############
+; DOGPILE CACHE
+; #############
+
+; Default cache dir for caches. Putting this into a ramdisk can boost performance.
+; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
+cache_dir = %(here)s/data
+
+; ***************************************
+; `repo_object` cache, default file based
+; ***************************************
+
+; `repo_object` cache settings for vcs methods for repositories
+rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+
+; cache auto-expires after N seconds
+; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+rc_cache.repo_object.expiration_time = 2592000
+
+; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
 
-## cache region for storing repo_objects cache
-rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
-## cache auto-expires after N seconds
-rc_cache.repo_object.expiration_time = 300
-## max size of LRU, old values will be discarded if the size of cache reaches max_size
-rc_cache.repo_object.max_size = 100
+; ***********************************************************
+; `repo_object` cache with redis backend
+; recommended for larger instance, and for better performance
+; ***********************************************************
+
+; `repo_object` cache settings for vcs methods for repositories
+#rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
+
+; cache auto-expires after N seconds
+; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+#rc_cache.repo_object.expiration_time = 2592000
+
+; redis_expiration_time needs to be greater then expiration_time
+#rc_cache.repo_object.arguments.redis_expiration_time = 3592000
+
+#rc_cache.repo_object.arguments.host = localhost
+#rc_cache.repo_object.arguments.port = 6379
+#rc_cache.repo_object.arguments.db = 5
+#rc_cache.repo_object.arguments.socket_timeout = 30
+; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
+#rc_cache.repo_object.arguments.distributed_lock = true
 
 
-################################
-### LOGGING CONFIGURATION   ####
-################################
+; #####################
+; LOGGING CONFIGURATION
+; #####################
 [loggers]
 keys = root, vcsserver
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic
 
-#############
-## LOGGERS ##
-#############
+; #######
+; LOGGERS
+; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_vcsserver]
 level = DEBUG
 handlers =
 qualname = vcsserver
 propagate = 1
 
 
-##############
-## HANDLERS ##
-##############
+; ########
+; HANDLERS
+; ########
 
 [handler_console]
 class = StreamHandler
-args = (sys.stderr,)
-level = DEBUG
+args = (sys.stderr, )
+level = INFO
 formatter = generic
 
-################
-## FORMATTERS ##
-################
+; ##########
+; FORMATTERS
+; ##########
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

pkgs/overlays.nix

@@ -1,47 +1,71 b''
 self: super: {
+
   # bump GIT version
   git = super.lib.overrideDerivation super.git (oldAttrs: {
-    name = "git-2.19.2";
+    name = "git-2.24.1";
     src = self.fetchurl {
-      url = "https://www.kernel.org/pub/software/scm/git/git-2.19.2.tar.xz";
-      sha256 = "1scbggzghkzzfqg4ky3qh7h9w87c3zya4ls5disz7dbx56is7sgw";
+      url = "https://www.kernel.org/pub/software/scm/git/git-2.24.1.tar.xz";
+      sha256 = "0ql5z31vgl7b785gwrf00m129mg7zi9pa65n12ij3mpxx3f28gvj";
     };
 
     # patches come from: https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/version-management/git-and-tools/git
     patches = [
       ./patches/git/docbook2texi.patch
       ./patches/git/git-sh-i18n.patch
       ./patches/git/ssh-path.patch
       ./patches/git/git-send-email-honor-PATH.patch
       ./patches/git/installCheck-path.patch
     ];
 
   });
 
+  libgit2rc = super.lib.overrideDerivation super.libgit2 (oldAttrs: {
+    name = "libgit2-0.28.2";
+    version = "0.28.2";
+
+    src = self.fetchFromGitHub {
+        owner = "libgit2";
+        repo = "libgit2";
+        rev = "v0.28.2";
+        sha256 = "0cm8fvs05rj0baigs2133q5a0sm3pa234y8h6hmwhl2bz9xq3k4b";
+    };
+
+    cmakeFlags = [ "-DTHREADSAFE=ON" "-DUSE_HTTPS=no"];
+
+    buildInputs = [
+        super.zlib
+        super.libssh2
+        super.openssl
+        super.curl
+    ];
+
+
+  });
+
   # Override subversion derivation to
   #  - activate python bindings
   subversion =
   let
     subversionWithPython = super.subversion.override {
       httpSupport = true;
       pythonBindings = true;
       python = self.python27Packages.python;
     };
   in
     super.lib.overrideDerivation subversionWithPython (oldAttrs: {
-      name = "subversion-1.10.2";
+      name = "subversion-1.12.2";
       src = self.fetchurl {
-        url = "https://archive.apache.org/dist/subversion/subversion-1.10.2.tar.gz";
-        sha256 = "0xv5z2bg0lw7057g913yc13f60nfj257wvmsq22pr33m4syf26sg";
+        url = "https://archive.apache.org/dist/subversion/subversion-1.12.2.tar.gz";
+        sha256 = "1wr1pklnq67xdzmf237zj6l1hg43yshfkbxvpvd5sv6r0dk7v4pl";
       };
 
       ## use internal lz4/utf8proc because it is stable and shipped with SVN
       configureFlags = oldAttrs.configureFlags ++ [
         " --with-lz4=internal"
         " --with-utf8proc=internal"
       ];
 
-
   });
 
+
 }

pkgs/patches/git/docbook2texi.patch

@@ -1,37 +1,38 b''
 This patch does two things: (1) use the right name for `docbook2texi',
 and (2) make sure `gitman.info' isn't produced since it's broken (duplicate
 node names).
 
 diff --git a/Documentation/Makefile b/Documentation/Makefile
+index 26a2342bea..ceccd67ebb 100644
 --- a/Documentation/Makefile
 +++ b/Documentation/Makefile
-@@ -122,7 +122,7 @@
+@@ -132,7 +132,7 @@ HTML_REPO = ../../git-htmldocs
 
  MAKEINFO = makeinfo
  INSTALL_INFO = install-info
 -DOCBOOK2X_TEXI = docbook2x-texi
 +DOCBOOK2X_TEXI = docbook2texi
  DBLATEX = dblatex
  ASCIIDOC_DBLATEX_DIR = /etc/asciidoc/dblatex
  DBLATEX_COMMON = -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty
-@@ -240,7 +240,7 @@
+@@ -250,7 +250,7 @@ man1: $(DOC_MAN1)
  man5: $(DOC_MAN5)
  man7: $(DOC_MAN7)
 
 -info: git.info gitman.info
 +info: git.info
 
  pdf: user-manual.pdf
 
-@@ -256,10 +256,9 @@
+@@ -266,10 +266,9 @@ install-man: man
 
  install-info: info
  	$(INSTALL) -d -m 755 $(DESTDIR)$(infodir)
 -	$(INSTALL) -m 644 git.info gitman.info $(DESTDIR)$(infodir)
 +	$(INSTALL) -m 644 git.info $(DESTDIR)$(infodir)
  	if test -r $(DESTDIR)$(infodir)/dir; then \
  	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) git.info ;\
 -	  $(INSTALL_INFO) --info-dir=$(DESTDIR)$(infodir) gitman.info ;\
  	else \
  	  echo "No directory found in $(DESTDIR)$(infodir)" >&2 ; \
  	fi

pkgs/patches/git/git-send-email-honor-PATH.patch

@@ -1,26 +1,28 b''
 diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt
+index 1afe9fc858..05dd7c3a90 100644
 --- a/Documentation/git-send-email.txt
 +++ b/Documentation/git-send-email.txt
-@@ -208,8 +208,7 @@ a password is obtained using 'git-credential'.
+@@ -215,8 +215,7 @@ a password is obtained using 'git-credential'.
  	specify a full pathname of a sendmail-like program instead;
  	the program must support the `-i` option.  Default value can
  	be specified by the `sendemail.smtpServer` configuration
 -	option; the built-in default is to search for `sendmail` in
 -	`/usr/sbin`, `/usr/lib` and $PATH if such program is
 +	option; the built-in default is to search in $PATH if such program is
  	available, falling back to `localhost` otherwise.
 
  --smtp-server-port=<port>::
 diff --git a/git-send-email.perl b/git-send-email.perl
+index 8eb63b5a2f..74a61d8213 100755
 --- a/git-send-email.perl
 +++ b/git-send-email.perl
-@@ -944,8 +944,7 @@ if (defined $reply_to) {
+@@ -956,8 +956,7 @@ sub expand_one_alias {
  }
 
  if (!defined $smtp_server) {
 -	my @sendmail_paths = qw( /usr/sbin/sendmail /usr/lib/sendmail );
 -	push @sendmail_paths, map {"$_/sendmail"} split /:/, $ENV{PATH};
 +	my @sendmail_paths = map {"$_/sendmail"} split /:/, $ENV{PATH};
  	foreach (@sendmail_paths) {
  		if (-x $_) {
  			$smtp_server = $_;

pkgs/patches/git/git-sh-i18n.patch

@@ -1,94 +1,23 b''
+diff --git a/git-sh-i18n.sh b/git-sh-i18n.sh
+index e1d917fd27..e90f8e1414 100644
 --- a/git-sh-i18n.sh
 +++ b/git-sh-i18n.sh
-@@ -15,87 +15,11 @@
- fi
- export TEXTDOMAINDIR
-
--# First decide what scheme to use...
--GIT_INTERNAL_GETTEXT_SH_SCHEME=fallthrough
--if test -n "$GIT_GETTEXT_POISON"
--then
--	GIT_INTERNAL_GETTEXT_SH_SCHEME=poison
--elif test -n "@@USE_GETTEXT_SCHEME@@"
--then
--	GIT_INTERNAL_GETTEXT_SH_SCHEME="@@USE_GETTEXT_SCHEME@@"
--elif test -n "$GIT_INTERNAL_GETTEXT_TEST_FALLBACKS"
--then
--	: no probing necessary
+@@ -26,7 +26,7 @@ then
+ elif test -n "$GIT_INTERNAL_GETTEXT_TEST_FALLBACKS"
+ then
+ 	: no probing necessary
 -elif type gettext.sh >/dev/null 2>&1
--then
--	# GNU libintl's gettext.sh
--	GIT_INTERNAL_GETTEXT_SH_SCHEME=gnu
--elif test "$(gettext -h 2>&1)" = "-h"
--then
--	# gettext binary exists but no gettext.sh. likely to be a gettext
--	# binary on a Solaris or something that is not GNU libintl and
--	# lack eval_gettext.
--	GIT_INTERNAL_GETTEXT_SH_SCHEME=gettext_without_eval_gettext
--fi
--export GIT_INTERNAL_GETTEXT_SH_SCHEME
--
--# ... and then follow that decision.
--case "$GIT_INTERNAL_GETTEXT_SH_SCHEME" in
--gnu)
--	# Use libintl's gettext.sh, or fall back to English if we can't.
++elif type @gettext@/bin/gettext.sh >/dev/null 2>&1
+ then
+ 	# GNU libintl's gettext.sh
+ 	GIT_INTERNAL_GETTEXT_SH_SCHEME=gnu
+@@ -43,7 +43,8 @@ export GIT_INTERNAL_GETTEXT_SH_SCHEME
+ case "$GIT_INTERNAL_GETTEXT_SH_SCHEME" in
+ gnu)
+ 	# Use libintl's gettext.sh, or fall back to English if we can't.
 -	. gettext.sh
--	;;
--gettext_without_eval_gettext)
--	# Solaris has a gettext(1) but no eval_gettext(1)
--	eval_gettext () {
--		gettext "$1" | (
--			export PATH $(git sh-i18n--envsubst --variables "$1");
--			git sh-i18n--envsubst "$1"
--		)
--	}
--
--	eval_ngettext () {
--		ngettext "$1" "$2" "$3" | (
--			export PATH $(git sh-i18n--envsubst --variables "$2");
--			git sh-i18n--envsubst "$2"
--		)
--	}
--	;;
--poison)
--	# Emit garbage so that tests that incorrectly rely on translatable
--	# strings will fail.
--	gettext () {
--		printf "%s" "# GETTEXT POISON #"
--	}
--
--	eval_gettext () {
--		printf "%s" "# GETTEXT POISON #"
--	}
--
--	eval_ngettext () {
--		printf "%s" "# GETTEXT POISON #"
--	}
--	;;
--*)
--	gettext () {
--		printf "%s" "$1"
--	}
--
--	eval_gettext () {
--		printf "%s" "$1" | (
--			export PATH $(git sh-i18n--envsubst --variables "$1");
--			git sh-i18n--envsubst "$1"
--		)
--	}
-+# GNU gettext
-+export GIT_INTERNAL_GETTEXT_SH_SCHEME=gnu
-+export PATH=@gettext@/bin:$PATH
-
--	eval_ngettext () {
--		(test "$3" = 1 && printf "%s" "$1" || printf "%s" "$2") | (
--			export PATH $(git sh-i18n--envsubst --variables "$2");
--			git sh-i18n--envsubst "$2"
--		)
--	}
--	;;
--esac
-+. @gettext@/bin/gettext.sh
-
- # Git-specific wrapper functions
- gettextln () {
++	. @gettext@/bin/gettext.sh
++	export PATH=@gettext@/bin:$PATH
+ 	;;
+ gettext_without_eval_gettext)
+ 	# Solaris has a gettext(1) but no eval_gettext(1)

pkgs/patches/git/installCheck-path.patch

@@ -1,12 +1,13 b''
 diff --git a/t/test-lib.sh b/t/test-lib.sh
+index 8665b0a9b6..8bb892b1af 100644
 --- a/t/test-lib.sh
 +++ b/t/test-lib.sh
-@@ -923,7 +923,7 @@
+@@ -1227,7 +1227,7 @@ elif test -n "$GIT_TEST_INSTALLED"
  then
  	GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path)  ||
  	error "Cannot run git from $GIT_TEST_INSTALLED."
--	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR:$PATH
+-	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH
 +	PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$GIT_BUILD_DIR:$PATH
  	GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH}
  else # normal case, use ../bin-wrappers only unless $with_dashes:
- 	git_bin_dir="$GIT_BUILD_DIR/bin-wrappers"
+ 	if test -n "$no_bin_wrappers"

pkgs/patches/git/ssh-path.patch

@@ -1,26 +1,26 b''
 diff --git a/connect.c b/connect.c
-index c3a014c5b..fbca3262b 100644
+index 4813f005ab..b3f12f3268 100644
 --- a/connect.c
 +++ b/connect.c
-@@ -1010,7 +1010,7 @@ static void fill_ssh_args(struct child_process *conn, const char *ssh_host,
+@@ -1183,7 +1183,7 @@ static void fill_ssh_args(struct child_process *conn, const char *ssh_host,
 
  		ssh = getenv("GIT_SSH");
  		if (!ssh)
 -			ssh = "ssh";
 +			ssh = "@ssh@";
  		variant = determine_ssh_variant(ssh, 0);
  	}
 
 diff --git a/git-gui/lib/remote_add.tcl b/git-gui/lib/remote_add.tcl
-index 480a6b30d..781720424 100644
+index 480a6b30d0..7817204241 100644
 --- a/git-gui/lib/remote_add.tcl
 +++ b/git-gui/lib/remote_add.tcl
 @@ -139,7 +139,7 @@ method _add {} {
  		# Parse the location
  		if { [regexp {(?:git\+)?ssh://([^/]+)(/.+)} $location xx host path]
  		     || [regexp {([^:][^:]+):(.+)} $location xx host path]} {
 -			set ssh ssh
 +			set ssh @ssh@
  			if {[info exists env(GIT_SSH)]} {
  				set ssh $env(GIT_SSH)
  			}

pkgs/python-packages-overrides.nix

@@ -1,60 +1,72 b''
 # Overrides for the generated python-packages.nix
 #
 # This function is intended to be used as an extension to the generated file
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
 { pkgs
 , basePythonPackages
 }:
 
 let
   sed = "sed -i";
 
 in
 
 self: super: {
 
+  "cffi" = super."cffi".override (attrs: {
+    buildInputs = [
+      pkgs.libffi
+    ];
+  });
+
   "gevent" = super."gevent".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       # NOTE: (marcink) odd requirements from gevent aren't set properly,
       # thus we need to inject psutil manually
       self."psutil"
     ];
   });
 
   "hgsubversion" = super."hgsubversion".override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.sqlite
       #basePythonPackages.sqlite3
       self.mercurial
     ];
   });
 
   "subvertpy" = super."subvertpy".override (attrs: {
     SVN_PREFIX = "${pkgs.subversion.dev}";
     propagatedBuildInputs = [
       pkgs.apr.dev
       pkgs.aprutil
       pkgs.subversion
     ];
   });
 
   "mercurial" = super."mercurial".override (attrs: {
     propagatedBuildInputs = [
       # self.python.modules.curses
     ];
   });
 
   "dulwich" = super."dulwich".override (attrs: {
     patches = [
       ./patches/dulwich/handle-dir-refs.patch
     ];
   });
 
+  "pygit2" = super."pygit2".override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      pkgs.libffi
+      pkgs.libgit2rc
+    ];
+  });
 
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;
 
 }

pkgs/python-packages.nix

@@ -1,948 +1,1090 b''
 # Generated by pip2nix 0.8.0.dev1
 # See https://github.com/johbo/pip2nix
 
 { pkgs, fetchurl, fetchgit, fetchhg }:
 
 self: super: {
   "atomicwrites" = super.buildPythonPackage {
-    name = "atomicwrites-1.2.1";
+    name = "atomicwrites-1.3.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/ac/ed/a311712ef6b4355035489f665e63e1a73f9eb371929e3c98e5efd451069e/atomicwrites-1.2.1.tar.gz";
-      sha256 = "1vmkbw9j0qammwxbxycrs39gvdg4lc2d4lk98kwf8ag2manyi6pc";
+      url = "https://files.pythonhosted.org/packages/ec/0f/cd484ac8820fed363b374af30049adc8fd13065720fd4f4c6be8a2309da7/atomicwrites-1.3.0.tar.gz";
+      sha256 = "19ngcscdf3jsqmpcxn6zl5b6anmsajb6izp1smcd1n02midl9abm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "attrs" = super.buildPythonPackage {
-    name = "attrs-18.2.0";
+    name = "attrs-19.3.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/0f/9e/26b1d194aab960063b266170e53c39f73ea0d0d3f5ce23313e0ec8ee9bdf/attrs-18.2.0.tar.gz";
-      sha256 = "0s9ydh058wmmf5v391pym877x4ahxg45dw6a0w4c7s5wgpigdjqh";
+      url = "https://files.pythonhosted.org/packages/98/c3/2c227e66b5e896e15ccdae2e00bbc69aa46e9a8ce8869cc5fa96310bf612/attrs-19.3.0.tar.gz";
+      sha256 = "0wky4h28n7xnr6xv69p9z6kv8bzn50d10c3drmd9ds8gawbcxdzp";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "backports.shutil-get-terminal-size" = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       sha256 = "107cmn7g3jnbkp826zlj8rrj19fam301qvaqf0f3905f5217lgki";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "beautifulsoup4" = super.buildPythonPackage {
     name = "beautifulsoup4-4.6.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/88/df/86bffad6309f74f3ff85ea69344a078fc30003270c8df6894fca7a3c72ff/beautifulsoup4-4.6.3.tar.gz";
       sha256 = "041dhalzjciw6qyzzq7a2k4h1yvyk76xigp35hv5ibnn448ydy4h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "cffi" = super.buildPythonPackage {
+    name = "cffi-1.12.3";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."pycparser"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/93/1a/ab8c62b5838722f29f3daffcc8d4bd61844aa9b5f437341cc890ceee483b/cffi-1.12.3.tar.gz";
+      sha256 = "0x075521fxwv0mfp4cqzk7lvmw4n94bjw601qkcv314z5s182704";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "configobj" = super.buildPythonPackage {
     name = "configobj-5.0.6";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626";
       sha256 = "0kqfrdfr14mw8yd8qwq14dv2xghpkjmd3yjsy8dfcbvpcc17xnxp";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  "configparser" = super.buildPythonPackage {
+    name = "configparser-4.0.2";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/16/4f/48975536bd488d3a272549eb795ac4a13a5f7fcdc8995def77fbef3532ee/configparser-4.0.2.tar.gz";
+      sha256 = "1priacxym85yjcf68hh38w55nqswaxp71ryjyfdk222kg9l85ln7";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
+  "contextlib2" = super.buildPythonPackage {
+    name = "contextlib2-0.6.0.post1";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/02/54/669207eb72e3d8ae8b38aa1f0703ee87a0e9f88f30d3c0a47bebdb6de242/contextlib2-0.6.0.post1.tar.gz";
+      sha256 = "0bhnr2ac7wy5l85ji909gyljyk85n92w8pdvslmrvc8qih4r1x01";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.psfl ];
+    };
+  };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."coverage"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       sha256 = "0k3np9ymh06yv1ib96sb6wfsxjkqhmik8qfsn119vnhga9ywc52a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "coverage" = super.buildPythonPackage {
-    name = "coverage-4.5.3";
+    name = "coverage-4.5.4";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/82/70/2280b5b29a0352519bb95ab0ef1ea942d40466ca71c53a2085bdeff7b0eb/coverage-4.5.3.tar.gz";
-      sha256 = "02f6m073qdispn96rc616hg0rnmw1pgqzw3bgxwiwza4zf9hirlx";
+      url = "https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz";
+      sha256 = "0p0j4di6h8k6ica7jwwj09azdcg4ycxq60i9qsskmsg94cd9yzg0";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   "decorator" = super.buildPythonPackage {
     name = "decorator-4.1.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bb/e0/f6e41e9091e130bf16d4437dabbac3993908e4d6485ecbc985ef1352db94/decorator-4.1.2.tar.gz";
       sha256 = "1d8npb11kxyi36mrvjdpcjij76l5zfyrz2f820brf0l0rcw4vdkw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   "dogpile.cache" = super.buildPythonPackage {
-    name = "dogpile.cache-0.7.1";
+    name = "dogpile.cache-0.9.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."decorator"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/84/3e/dbf1cfc5228f1d3dca80ef714db2c5aaec5cd9efaf54d7e3daef6bc48b19/dogpile.cache-0.7.1.tar.gz";
-      sha256 = "0caazmrzhnfqb5yrp8myhw61ny637jj69wcngrpbvi31jlcpy6v9";
+      url = "https://files.pythonhosted.org/packages/ac/6a/9ac405686a94b7f009a20a50070a5786b0e1aedc707b88d40d0c4b51a82e/dogpile.cache-0.9.0.tar.gz";
+      sha256 = "0sr1fn6b4k5bh0cscd9yi8csqxvj4ngzildav58x5p694mc86j5k";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dogpile.core" = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       sha256 = "0xpdvg4kr1isfkrh1rfsh7za4q5a5s6l2kf9wpvndbwf3aqjyrdy";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "dulwich" = super.buildPythonPackage {
     name = "dulwich-0.13.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz";
       sha256 = "0f1jwvrh549c4rgavkn3wizrch904s73s4fmrxykxy9cw8s57lwf";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "enum34" = super.buildPythonPackage {
     name = "enum34-1.1.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
       sha256 = "1cgm5ng2gcfrkrm3hc22brl6chdmv67b9zvva9sfs7gn7dwc9n4a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "funcsigs" = super.buildPythonPackage {
     name = "funcsigs-1.0.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/94/4a/db842e7a0545de1cdb0439bb80e6e42dfe82aaeaadd4072f2263a4fbed23/funcsigs-1.0.2.tar.gz";
       sha256 = "0l4g5818ffyfmfs1a924811azhjj8ax9xd1cffr1mzd3ycn0zfx7";
     };
     meta = {
       license = [ { fullName = "ASL"; } pkgs.lib.licenses.asl20 ];
     };
   };
   "gevent" = super.buildPythonPackage {
     name = "gevent-1.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."greenlet"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/ed/27/6c49b70808f569b66ec7fac2e78f076e9b204db9cf5768740cff3d5a07ae/gevent-1.4.0.tar.gz";
       sha256 = "1lchr4akw2jkm5v4kz7bdm4wv3knkfhbfn9vkkz4s5yrkcxzmdqy";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gprof2dot" = super.buildPythonPackage {
     name = "gprof2dot-2017.9.19";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/36/f977122502979f3dfb50704979c9ed70e6b620787942b089bf1af15f5aba/gprof2dot-2017.9.19.tar.gz";
       sha256 = "17ih23ld2nzgc3xwgbay911l6lh96jp1zshmskm17n1gg2i7mg6f";
     };
     meta = {
       license = [ { fullName = "GNU Lesser General Public License v3 or later (LGPLv3+)"; } { fullName = "LGPL"; } ];
     };
   };
   "greenlet" = super.buildPythonPackage {
     name = "greenlet-0.4.15";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/f8/e8/b30ae23b45f69aa3f024b46064c0ac8e5fcb4f22ace0dca8d6f9c8bbe5e7/greenlet-0.4.15.tar.gz";
       sha256 = "1g4g1wwc472ds89zmqlpyan3fbnzpa8qm48z3z1y6mlk44z485ll";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "gunicorn" = super.buildPythonPackage {
     name = "gunicorn-19.9.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/47/52/68ba8e5e8ba251e54006a49441f7ccabca83b6bef5aedacb4890596c7911/gunicorn-19.9.0.tar.gz";
       sha256 = "1wzlf4xmn6qjirh5w81l6i6kqjnab1n1qqkh7zsj1yb6gh4n49ps";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "hg-evolve" = super.buildPythonPackage {
-    name = "hg-evolve-8.5.1";
+    name = "hg-evolve-9.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/e3/ce/6594aa403e3464831d4daf20e45fd2e3ef553d968ac13d2c7fa791d4eedd/hg-evolve-8.5.1.tar.gz";
-      sha256 = "09avqn7c1biz97vb1zw91q6nfzydpcqv43mgpfrj7ywp0fscfgf3";
+      url = "https://files.pythonhosted.org/packages/20/36/5a6655975aa0c663be91098d31a0b24841acad44fe896aa2bdee77c6b883/hg-evolve-9.1.0.tar.gz";
+      sha256 = "1mna81cmzxxn7s2nwz3g1xgdjlcc1axkvfmwg7gjqghwn3pdraps";
     };
     meta = {
       license = [ { fullName = "GPLv2+"; } ];
     };
   };
   "hgsubversion" = super.buildPythonPackage {
     name = "hgsubversion-1.9.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."mercurial"
       self."subvertpy"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a3/53/6d205e641f3e09abcf1ddaed66e5e4b20da22d0145566d440a02c9e35f0d/hgsubversion-1.9.3.tar.gz";
       sha256 = "0nymcjlch8c4zjbncrs30p2nrbylsf25g3h6mr0zzzxr141h3sig";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   "hupper" = super.buildPythonPackage {
-    name = "hupper-1.6.1";
+    name = "hupper-1.9.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/85/d9/e005d357b11249c5d70ddf5b7adab2e4c0da4e8b0531ff146917a04fe6c0/hupper-1.6.1.tar.gz";
-      sha256 = "0d3cvkc8ssgwk54wvhbifj56ry97qi10pfzwfk8vwzzcikbfp3zy";
+      url = "https://files.pythonhosted.org/packages/09/3a/4f215659f31eeffe364a984dba486bfa3907bfcc54b7013bdfe825cebb5f/hupper-1.9.1.tar.gz";
+      sha256 = "0pyg879fv9mbwlnbzw2a3234qqycqs9l97h5mpkmk0bvxhi2471v";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "importlib-metadata" = super.buildPythonPackage {
+    name = "importlib-metadata-0.23";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."zipp"
+      self."contextlib2"
+      self."configparser"
+      self."pathlib2"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/5d/44/636bcd15697791943e2dedda0dbe098d8530a38d113b202817133e0b06c0/importlib_metadata-0.23.tar.gz";
+      sha256 = "09mdqdfv5rdrwz80jh9m379gxmvk2vhjfz0fg53hid00icvxf65a";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
   "ipdb" = super.buildPythonPackage {
     name = "ipdb-0.12";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."ipython"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/6d/43/c3c2e866a8803e196d6209595020a4a6db1a3c5d07c01455669497ae23d0/ipdb-0.12.tar.gz";
       sha256 = "1khr2n7xfy8hg65kj1bsrjq9g7656pp0ybfa8abpbzpdawji3qnw";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython" = super.buildPythonPackage {
     name = "ipython-5.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
       self."decorator"
       self."pickleshare"
       self."simplegeneric"
       self."traitlets"
       self."prompt-toolkit"
       self."pygments"
       self."pexpect"
       self."backports.shutil-get-terminal-size"
       self."pathlib2"
       self."pexpect"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       sha256 = "0qdrf6aj9kvjczd5chj1my8y2iq09am9l8bb2a1334a52d76kx3y";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ipython-genutils" = super.buildPythonPackage {
     name = "ipython-genutils-0.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e8/69/fbeffffc05236398ebfcfb512b6d2511c622871dca1746361006da310399/ipython_genutils-0.2.0.tar.gz";
       sha256 = "1a4bc9y8hnvq6cp08qs4mckgm6i6ajpndp4g496rvvzcfmp12bpb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "mako" = super.buildPythonPackage {
-    name = "mako-1.0.7";
+    name = "mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."markupsafe"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/eb/f3/67579bb486517c0d49547f9697e36582cd19dafb5df9e687ed8e22de57fa/Mako-1.0.7.tar.gz";
-      sha256 = "1bi5gnr8r8dva06qpyx4kgjc6spm2k1y908183nbbaylggjzs0jf";
+      url = "https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz";
+      sha256 = "0jqa3qfpykyn4fmkn0kh6043sfls7br8i2bsdbccazcvk9cijsd3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "markupsafe" = super.buildPythonPackage {
-    name = "markupsafe-1.1.0";
+    name = "markupsafe-1.1.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/ac/7e/1b4c2e05809a4414ebce0892fe1e32c14ace86ca7d50c70f00979ca9b3a3/MarkupSafe-1.1.0.tar.gz";
-      sha256 = "1lxirjypbdd3l9jl4vliilhfnhy7c7f2vlldqg1b0i74khn375sf";
+      url = "https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz";
+      sha256 = "0sqipg4fk7xbixqd8kq6rlkxj664d157bdwbh93farcphf92x1r9";
     };
     meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal ];
+      license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd3 ];
     };
   };
   "mercurial" = super.buildPythonPackage {
-    name = "mercurial-4.9.1";
+    name = "mercurial-5.1.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/60/58/a1c52d5f5c0b755e231faf7c4f507dc51fe26d979d36346bc9d28f4f8a75/mercurial-4.9.1.tar.gz";
-      sha256 = "0iybbkd9add066729zg01kwz5hhc1s6lhp9rrnsmzq6ihyxj3p8v";
+      url = "https://files.pythonhosted.org/packages/22/39/e1a95f6048aa0785b82f5faad8281ae7320894a635cb4a57e19479639c92/mercurial-5.1.1.tar.gz";
+      sha256 = "17z42rfjdkrks4grzgac66nfh285zf1pwxd2zwx1p71pw2jqpz1m";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "mock" = super.buildPythonPackage {
-    name = "mock-1.0.1";
+    name = "mock-3.0.5";
     doCheck = false;
+    propagatedBuildInputs = [
+      self."six"
+      self."funcsigs"
+    ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz";
-      sha256 = "0kzlsbki6q0awf89rc287f3aj8x431lrajf160a70z0ikhnxsfdq";
+      url = "https://files.pythonhosted.org/packages/2e/ab/4fe657d78b270aa6a32f027849513b829b41b0f28d9d8d7f8c3d29ea559a/mock-3.0.5.tar.gz";
+      sha256 = "1hrp6j0yrx2xzylfv02qa8kph661m6yq4p0mc8fnimch9j4psrc3";
     };
     meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal ];
+      license = [ pkgs.lib.licenses.bsdOriginal { fullName = "OSI Approved :: BSD License"; } ];
     };
   };
   "more-itertools" = super.buildPythonPackage {
     name = "more-itertools-5.0.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/dd/26/30fc0d541d9fdf55faf5ba4b0fd68f81d5bd2447579224820ad525934178/more-itertools-5.0.0.tar.gz";
       sha256 = "1r12cm6mcdwdzz7d47a6g4l437xsvapdlgyhqay3i2nrlv03da9q";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "msgpack-python" = super.buildPythonPackage {
     name = "msgpack-python-0.5.6";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/20/6eca772d1a5830336f84aca1d8198e5a3f4715cd1c7fc36d3cc7f7185091/msgpack-python-0.5.6.tar.gz";
       sha256 = "16wh8qgybmfh4pjp8vfv78mdlkxfmcasg78lzlnm6nslsfkci31p";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  "packaging" = super.buildPythonPackage {
+    name = "packaging-19.2";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."pyparsing"
+      self."six"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/5a/2f/449ded84226d0e2fda8da9252e5ee7731bdf14cd338f622dfcd9934e0377/packaging-19.2.tar.gz";
+      sha256 = "0izwlz9h0bw171a1chr311g2y7n657zjaf4mq4rgm8pp9lbj9f98";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal { fullName = "BSD or Apache License, Version 2.0"; } pkgs.lib.licenses.asl20 ];
+    };
+  };
   "pastedeploy" = super.buildPythonPackage {
     name = "pastedeploy-2.0.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/19/a0/5623701df7e2478a68a1b685d1a84518024eef994cde7e4da8449a31616f/PasteDeploy-2.0.1.tar.gz";
       sha256 = "02imfbbx1mi2h546f3sr37m47dk9qizaqhzzlhx8bkzxa6fzn8yl";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pathlib2" = super.buildPythonPackage {
-    name = "pathlib2-2.3.4";
+    name = "pathlib2-2.3.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."scandir"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/b5/f4/9c7cc726ece2498b6c8b62d3262aa43f59039b953fe23c9964ac5e18d40b/pathlib2-2.3.4.tar.gz";
-      sha256 = "1y0f9rkm1924zrc5dn4bwxlhgdkbml82lkcc28l5rgmr7d918q24";
+      url = "https://files.pythonhosted.org/packages/94/d8/65c86584e7e97ef824a1845c72bbe95d79f5b306364fa778a3c3e401b309/pathlib2-2.3.5.tar.gz";
+      sha256 = "0s4qa8c082fdkb17izh4mfgwrjd1n5pya18wvrbwqdvvb5xs9nbc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pexpect" = super.buildPythonPackage {
     name = "pexpect-4.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."ptyprocess"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/1c/b1/362a0d4235496cb42c33d1d8732b5e2c607b0129ad5fdd76f5a583b9fcb3/pexpect-4.7.0.tar.gz";
       sha256 = "1sv2rri15zwhds85a4kamwh9pj49qcxv7m4miyr4jfpfwv81yb4y";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   "pickleshare" = super.buildPythonPackage {
     name = "pickleshare-0.7.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."pathlib2"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz";
       sha256 = "1jmghg3c53yp1i8cm6pcrm280ayi8621rwyav9fac7awjr3kss47";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster" = super.buildPythonPackage {
     name = "plaster-1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/37/e1/56d04382d718d32751017d32f351214384e529b794084eee20bb52405563/plaster-1.0.tar.gz";
       sha256 = "1hy8k0nv2mxq94y5aysk6hjk9ryb4bsd13g83m60hcyzxz3wflc3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "plaster-pastedeploy" = super.buildPythonPackage {
     name = "plaster-pastedeploy-0.7";
     doCheck = false;
     propagatedBuildInputs = [
       self."pastedeploy"
       self."plaster"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/99/69/2d3bc33091249266a1bd3cf24499e40ab31d54dffb4a7d76fe647950b98c/plaster_pastedeploy-0.7.tar.gz";
       sha256 = "1zg7gcsvc1kzay1ry5p699rg2qavfsxqwl17mqxzr0gzw6j9679r";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pluggy" = super.buildPythonPackage {
-    name = "pluggy-0.11.0";
+    name = "pluggy-0.13.1";
     doCheck = false;
+    propagatedBuildInputs = [
+      self."importlib-metadata"
+    ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/0d/a1/862ab336e8128fde20981d2c1aa8506693412daf5083b1911d539412676b/pluggy-0.11.0.tar.gz";
-      sha256 = "10511a54dvafw1jrk75mrhml53c7b7w4yaw7241696lc2hfvr895";
+      url = "https://files.pythonhosted.org/packages/f8/04/7a8542bed4b16a65c2714bf76cf5a0b026157da7f75e87cc88774aa10b14/pluggy-0.13.1.tar.gz";
+      sha256 = "1c35qyhvy27q9ih9n899f3h4sdnpgq027dbiilly2qb5cvgarchm";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "prompt-toolkit" = super.buildPythonPackage {
-    name = "prompt-toolkit-1.0.16";
+    name = "prompt-toolkit-1.0.18";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."wcwidth"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/f1/03/bb36771dc9fa7553ac4bdc639a9ecdf6fda0ff4176faf940d97e3c16e41d/prompt_toolkit-1.0.16.tar.gz";
-      sha256 = "1d65hm6nf0cbq0q0121m60zzy4s1fpg9fn761s1yxf08dridvkn1";
+      url = "https://files.pythonhosted.org/packages/c5/64/c170e5b1913b540bf0c8ab7676b21fdd1d25b65ddeb10025c6ca43cccd4c/prompt_toolkit-1.0.18.tar.gz";
+      sha256 = "09h1153wgr5x2ny7ds0w2m81n3bb9j8hjb8sjfnrg506r01clkyx";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "psutil" = super.buildPythonPackage {
-    name = "psutil-5.5.1";
+    name = "psutil-5.6.5";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/c7/01/7c30b247cdc5ba29623faa5c8cf1f1bbf7e041783c340414b0ed7e067c64/psutil-5.5.1.tar.gz";
-      sha256 = "045qaqvn6k90bj5bcy259yrwcd2afgznaav3sfhphy9b8ambzkkj";
+      url = "https://files.pythonhosted.org/packages/03/9a/95c4b3d0424426e5fd94b5302ff74cea44d5d4f53466e1228ac8e73e14b4/psutil-5.6.5.tar.gz";
+      sha256 = "0isil5jxwwd8awz54qk28rpgjg43i5l6yl70g40vxwa4r4m56lfh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "ptyprocess" = super.buildPythonPackage {
     name = "ptyprocess-0.6.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7d/2d/e4b8733cf79b7309d84c9081a4ab558c89d8c89da5961bf4ddb050ca1ce0/ptyprocess-0.6.0.tar.gz";
       sha256 = "1h4lcd3w5nrxnsk436ar7fwkiy5rfn5wj2xwy9l0r4mdqnf2jgwj";
     };
     meta = {
       license = [  ];
     };
   };
   "py" = super.buildPythonPackage {
-    name = "py-1.6.0";
+    name = "py-1.8.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/4f/38/5f427d1eedae73063ce4da680d2bae72014995f9fdeaa57809df61c968cd/py-1.6.0.tar.gz";
-      sha256 = "1wcs3zv9wl5m5x7p16avqj2gsrviyb23yvc3pr330isqs0sh98q6";
+      url = "https://files.pythonhosted.org/packages/f1/5a/87ca5909f400a2de1561f1648883af74345fe96349f34f737cdfc94eba8c/py-1.8.0.tar.gz";
+      sha256 = "0lsy1gajva083pzc7csj1cvbmminb7b4l6a0prdzyb3fd829nqyw";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "pycparser" = super.buildPythonPackage {
+    name = "pycparser-2.19";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a/pycparser-2.19.tar.gz";
+      sha256 = "1cr5dcj9628lkz1qlwq3fv97c25363qppkmcayqvd05dpy573259";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  };
+  "pygit2" = super.buildPythonPackage {
+    name = "pygit2-0.28.2";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."cffi"
+      self."six"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/4c/64/88c2a4eb2d22ca1982b364f41ff5da42d61de791d7eb68140e7f8f7eb721/pygit2-0.28.2.tar.gz";
+      sha256 = "11kzj5mjkspvplnpdb6bj8dcj6rgmkk986k8hjcklyg5yaxkz32d";
+    };
+    meta = {
+      license = [ { fullName = "GPLv2 with linking exception"; } ];
+    };
+  };
   "pygments" = super.buildPythonPackage {
     name = "pygments-2.4.2";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/ae/26808275fc76bf2832deb10d3a3ed3107bc4de01b85dcccbe525f2cd6d1e/Pygments-2.4.2.tar.gz";
       sha256 = "15v2sqm5g12bqa0c7wikfh9ck2nl97ayizy1hpqhmws5gqalq748";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
+  "pyparsing" = super.buildPythonPackage {
+    name = "pyparsing-2.4.5";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/00/32/8076fa13e832bb4dcff379f18f228e5a53412be0631808b9ca2610c0f566/pyparsing-2.4.5.tar.gz";
+      sha256 = "0fk8gsybiw1gm146mkjdjvaajwh20xwvpv4j7syh2zrnpq0j19jc";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "pyramid" = super.buildPythonPackage {
     name = "pyramid-1.10.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."hupper"
       self."plaster"
       self."plaster-pastedeploy"
       self."setuptools"
       self."translationstring"
       self."venusian"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."repoze.lru"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/c2/43/1ae701c9c6bb3a434358e678a5e72c96e8aa55cf4cb1d2fa2041b5dd38b7/pyramid-1.10.4.tar.gz";
       sha256 = "0rkxs1ajycg2zh1c94xlmls56mx5m161sn8112skj0amza6cn36q";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pyramid-mako" = super.buildPythonPackage {
-    name = "pyramid-mako-1.0.2";
+    name = "pyramid-mako-1.1.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pyramid"
       self."mako"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
-      sha256 = "18gk2vliq8z4acblsl6yzgbvnr9rlxjlcqir47km7kvlk1xri83d";
+      url = "https://files.pythonhosted.org/packages/63/7b/5e2af68f675071a6bad148c1c393928f0ef5fcd94e95cbf53b89d6471a83/pyramid_mako-1.1.0.tar.gz";
+      sha256 = "1qj0m091mnii86j2q1d82yir22nha361rvhclvg3s70z8iiwhrh0";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "pytest" = super.buildPythonPackage {
-    name = "pytest-3.8.2";
+    name = "pytest-4.6.5";
     doCheck = false;
     propagatedBuildInputs = [
       self."py"
       self."six"
-      self."setuptools"
+      self."packaging"
       self."attrs"
-      self."more-itertools"
       self."atomicwrites"
       self."pluggy"
+      self."importlib-metadata"
+      self."wcwidth"
       self."funcsigs"
       self."pathlib2"
+      self."more-itertools"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/5f/d2/7f77f406ac505abda02ab4afb50d06ebf304f6ea42fca34f8f37529106b2/pytest-3.8.2.tar.gz";
-      sha256 = "18nrwzn61kph2y6gxwfz9ms68rfvr9d4vcffsxng9p7jk9z18clk";
+      url = "https://files.pythonhosted.org/packages/2a/c6/1d1f32f6a5009900521b12e6560fb6b7245b0d4bc3fb771acd63d10e30e1/pytest-4.6.5.tar.gz";
+      sha256 = "0iykwwfp4h181nd7rsihh2120b0rkawlw7rvbl19sgfspncr3hwg";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-cov" = super.buildPythonPackage {
-    name = "pytest-cov-2.6.0";
+    name = "pytest-cov-2.7.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."coverage"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/d9/e2/58f90a316fbd94dd50bf5c826a23f3f5d079fb3cc448c1e9f0e3c33a3d2a/pytest-cov-2.6.0.tar.gz";
-      sha256 = "0qnpp9y3ygx4jk4pf5ad71fh2skbvnr6gl54m7rg5qysnx4g0q73";
+      url = "https://files.pythonhosted.org/packages/bb/0f/3db7ff86801883b21d5353b258c994b1b8e2abbc804e2273b8d0fd19004b/pytest-cov-2.7.1.tar.gz";
+      sha256 = "0filvmmyqm715azsl09ql8hy2x7h286n6d8z5x42a1wpvvys83p0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   "pytest-profiling" = super.buildPythonPackage {
-    name = "pytest-profiling-1.3.0";
+    name = "pytest-profiling-1.7.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."pytest"
       self."gprof2dot"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/f5/34/4626126e041a51ef50a80d0619519b18d20aef249aac25b0d0fdd47e57ee/pytest-profiling-1.3.0.tar.gz";
-      sha256 = "08r5afx5z22yvpmsnl91l4amsy1yxn8qsmm61mhp06mz8zjs51kb";
+      url = "https://files.pythonhosted.org/packages/39/70/22a4b33739f07f1732a63e33bbfbf68e0fa58cfba9d200e76d01921eddbf/pytest-profiling-1.7.0.tar.gz";
+      sha256 = "0abz9gi26jpcfdzgsvwad91555lpgdc8kbymicmms8k2fqa8z4wk";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-runner" = super.buildPythonPackage {
-    name = "pytest-runner-4.2";
+    name = "pytest-runner-5.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/9e/b7/fe6e8f87f9a756fd06722216f1b6698ccba4d269eac6329d9f0c441d0f93/pytest-runner-4.2.tar.gz";
-      sha256 = "1gkpyphawxz38ni1gdq1fmwyqcg02m7ypzqvv46z06crwdxi2gyj";
+      url = "https://files.pythonhosted.org/packages/d9/6d/4b41a74b31720e25abd4799be72d54811da4b4d0233e38b75864dcc1f7ad/pytest-runner-5.1.tar.gz";
+      sha256 = "0ykfcnpp8c22winj63qzc07l5axwlc9ikl8vn05sc32gv3417815";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "pytest-sugar" = super.buildPythonPackage {
-    name = "pytest-sugar-0.9.1";
+    name = "pytest-sugar-0.9.2";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
       self."termcolor"
+      self."packaging"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/3e/6a/a3f909083079d03bde11d06ab23088886bbe25f2c97fbe4bb865e2bf05bc/pytest-sugar-0.9.1.tar.gz";
-      sha256 = "0b4av40dv30727m54v211r0nzwjp2ajkjgxix6j484qjmwpw935b";
+      url = "https://files.pythonhosted.org/packages/55/59/f02f78d1c80f7e03e23177f60624c8106d4f23d124c921df103f65692464/pytest-sugar-0.9.2.tar.gz";
+      sha256 = "1asq7yc4g8bx2sn7yy974mhc9ywvaihasjab4inkirdwn9s7mn7w";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "pytest-timeout" = super.buildPythonPackage {
-    name = "pytest-timeout-1.3.2";
+    name = "pytest-timeout-1.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytest"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/8c/3e/1b6a319d12ae7baa3acb7c18ff2c8630a09471a0319d43535c683b4d03eb/pytest-timeout-1.3.2.tar.gz";
-      sha256 = "09wnmzvnls2mnsdz7x3c3sk2zdp6jl4dryvyj5i8hqz16q2zq5qi";
+      url = "https://files.pythonhosted.org/packages/13/48/7a166eaa29c1dca6cc253e3ba5773ff2e4aa4f567c1ea3905808e95ac5c1/pytest-timeout-1.3.3.tar.gz";
+      sha256 = "1cczcjhw4xx5sjkhxlhc5c1bkr7x6fcyx12wrnvwfckshdvblc2a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
+  "redis" = super.buildPythonPackage {
+    name = "redis-3.3.11";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/06/ca/00557c74279d2f256d3c42cabf237631355f3a132e4c74c2000e6647ad98/redis-3.3.11.tar.gz";
+      sha256 = "1hicqbi5xl92hhml82awrr2rxl9jar5fp8nbcycj9qgmsdwc43wd";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "repoze.lru" = super.buildPythonPackage {
     name = "repoze.lru-0.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz";
       sha256 = "0xzz1aw2smy8hdszrq8yhnklx6w1r1mf55061kalw3iq35gafa84";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "rhodecode-vcsserver" = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.17.4";
+    name = "rhodecode-vcsserver-4.18.0";
     buildInputs = [
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
       self."configobj"
     ];
     doCheck = true;
     propagatedBuildInputs = [
       self."configobj"
       self."dogpile.cache"
       self."dogpile.core"
       self."decorator"
       self."dulwich"
       self."hgsubversion"
       self."hg-evolve"
       self."mako"
       self."markupsafe"
       self."mercurial"
       self."msgpack-python"
       self."pastedeploy"
       self."pyramid"
       self."pyramid-mako"
+      self."pygit2"
       self."repoze.lru"
+      self."redis"
       self."simplejson"
       self."subprocess32"
       self."subvertpy"
       self."six"
       self."translationstring"
       self."webob"
       self."zope.deprecation"
       self."zope.interface"
       self."gevent"
       self."greenlet"
       self."gunicorn"
       self."waitress"
       self."ipdb"
       self."ipython"
       self."pytest"
       self."py"
       self."pytest-cov"
       self."pytest-sugar"
       self."pytest-runner"
       self."pytest-profiling"
       self."pytest-timeout"
       self."gprof2dot"
       self."mock"
       self."cov-core"
       self."coverage"
       self."webtest"
       self."beautifulsoup4"
     ];
     src = ./.;
     meta = {
       license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
     };
   };
   "scandir" = super.buildPythonPackage {
     name = "scandir-1.10.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz";
       sha256 = "1bkqwmf056pkchf05ywbnf659wqlp6lljcdb0y88wr9f0vv32ijd";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
     };
   };
   "setproctitle" = super.buildPythonPackage {
     name = "setproctitle-1.1.10";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz";
       sha256 = "163kplw9dcrw0lffq1bvli5yws3rngpnvrxrzdw89pbphjjvg0v2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "setuptools" = super.buildPythonPackage {
-    name = "setuptools-41.0.1";
+    name = "setuptools-44.0.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/1d/64/a18a487b4391a05b9c7f938b94a16d80305bf0369c6b0b9509e86165e1d3/setuptools-41.0.1.zip";
-      sha256 = "04sns22y2hhsrwfy1mha2lgslvpjsjsz8xws7h2rh5a7ylkd28m2";
+      url = "https://files.pythonhosted.org/packages/b0/f3/44da7482ac6da3f36f68e253cb04de37365b3dba9036a3c70773b778b485/setuptools-44.0.0.zip";
+      sha256 = "025h5cnxcmda1893l6i12hrwdvs1n8r31qs6q4pkif2v7rrggfp5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "simplegeneric" = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "simplejson" = super.buildPythonPackage {
     name = "simplejson-3.16.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz";
       sha256 = "19cws1syk8jzq2pw43878dv6fjkb0ifvjpx0i9aajix6kc9jkwxi";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   "six" = super.buildPythonPackage {
     name = "six-1.11.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz";
       sha256 = "1scqzwc51c875z23phj48gircqjgnn3af8zy2izjwmnlxrxsgs3h";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "subprocess32" = super.buildPythonPackage {
     name = "subprocess32-3.5.4";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz";
       sha256 = "17f7mvwx2271s1wrl0qac3wjqqnrqag866zs3qc8v5wp0k43fagb";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   "subvertpy" = super.buildPythonPackage {
     name = "subvertpy-0.10.1";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/76/99fa82affce75f5ac0f7dbe513796c3f37311ace0c68e1b063683b4f9b99/subvertpy-0.10.1.tar.gz";
       sha256 = "061ncy9wjz3zyv527avcrdyk0xygyssyy7p1644nhzhwp8zpybij";
     };
     meta = {
       license = [ pkgs.lib.licenses.lgpl21Plus pkgs.lib.licenses.gpl2Plus ];
     };
   };
   "termcolor" = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       sha256 = "0fv1vq14rpqwgazxg4981904lfyp84mnammw7y046491cv76jv8x";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "traitlets" = super.buildPythonPackage {
-    name = "traitlets-4.3.2";
+    name = "traitlets-4.3.3";
     doCheck = false;
     propagatedBuildInputs = [
       self."ipython-genutils"
       self."six"
       self."decorator"
       self."enum34"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/a5/98/7f5ef2fe9e9e071813aaf9cb91d1a732e0a68b6c44a32b38cb8e14c3f069/traitlets-4.3.2.tar.gz";
-      sha256 = "0dbq7sx26xqz5ixs711k5nc88p8a0nqyz6162pwks5dpcz9d4jww";
+      url = "https://files.pythonhosted.org/packages/75/b0/43deb021bc943f18f07cbe3dac1d681626a48997b7ffa1e7fb14ef922b21/traitlets-4.3.3.tar.gz";
+      sha256 = "1xsrwgivpkxlbr4dfndfsi098s29yqgswgjc1qqn69yxklvfw8yh";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   "translationstring" = super.buildPythonPackage {
     name = "translationstring-1.3";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       sha256 = "0bdpcnd9pv0131dl08h4zbcwmgc45lyvq3pa224xwan5b3x4rr2f";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   "venusian" = super.buildPythonPackage {
     name = "venusian-1.2.0";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/7e/6f/40a9d43ac77cb51cb62be5b5662d170f43f8037bdc4eab56336c4ca92bb7/venusian-1.2.0.tar.gz";
       sha256 = "0ghyx66g8ikx9nx1mnwqvdcqm11i1vlq0hnvwl50s48bp22q5v34";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   "waitress" = super.buildPythonPackage {
-    name = "waitress-1.3.0";
+    name = "waitress-1.3.1";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/43/50/9890471320d5ad22761ae46661cf745f487b1c8c4ec49352b99e1078b970/waitress-1.3.0.tar.gz";
-      sha256 = "09j5dzbbcxib7vdskhx39s1qsydlr4n2p2png71d7mjnr9pnwajf";
+      url = "https://files.pythonhosted.org/packages/a6/e6/708da7bba65898e5d759ade8391b1077e49d07be0b0223c39f5be04def56/waitress-1.3.1.tar.gz";
+      sha256 = "1iysl8ka3l4cdrr0r19fh1cv28q41mwpvgsb81ji7k4shkb0k3i7";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "wcwidth" = super.buildPythonPackage {
     name = "wcwidth-0.1.7";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
       sha256 = "0pn6dflzm609m4r3i8ik5ni9ijjbb5fa3vg1n7hn6vkd49r77wrx";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webob" = super.buildPythonPackage {
     name = "webob-1.8.5";
     doCheck = false;
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/9d/1a/0c89c070ee2829c934cb6c7082287c822e28236a4fcf90063e6be7c35532/WebOb-1.8.5.tar.gz";
       sha256 = "11khpzaxc88q31v25ic330gsf56fwmbdc9b30br8mvp0fmwspah5";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   "webtest" = super.buildPythonPackage {
     name = "webtest-2.0.33";
     doCheck = false;
     propagatedBuildInputs = [
       self."six"
       self."webob"
       self."waitress"
       self."beautifulsoup4"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/a8/b0/ffc9413b637dbe26e291429bb0f6ed731e518d0cd03da28524a8fe2e8a8f/WebTest-2.0.33.tar.gz";
       sha256 = "1l3z0cwqslsf4rcrhi2gr8kdfh74wn2dw76376i4g9i38gz8wd21";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "zipp" = super.buildPythonPackage {
+    name = "zipp-0.6.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."more-itertools"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/57/dd/585d728479d97d25aeeb9aa470d36a4ad8d0ba5610f84e14770128ce6ff7/zipp-0.6.0.tar.gz";
+      sha256 = "13ndkf7vklw978a4gdl1yfvn8hch28429a0iam67sg4nrp5v261p";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "zope.deprecation" = super.buildPythonPackage {
     name = "zope.deprecation-4.4.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/34/da/46e92d32d545dd067b9436279d84c339e8b16de2ca393d7b892bc1e1e9fd/zope.deprecation-4.4.0.tar.gz";
       sha256 = "1pz2cv7gv9y1r3m0bdv7ks1alagmrn5msm5spwdzkb2by0w36i8d";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
   "zope.interface" = super.buildPythonPackage {
     name = "zope.interface-4.6.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."setuptools"
     ];
     src = fetchurl {
       url = "https://files.pythonhosted.org/packages/4e/d0/c9d16bd5b38de44a20c6dc5d5ed80a49626fafcb3db9f9efdc2a19026db6/zope.interface-4.6.0.tar.gz";
       sha256 = "1rgh2x3rcl9r0v0499kf78xy86rnmanajf4ywmqb943wpk50sg8v";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpl21 ];
     };
   };
 
 ### Test requirements
 
   
 }

pkgs/shell-generate.nix

@@ -1,41 +1,42 b''
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 }:
 
 with pkgs.lib;
 
 let _pythonPackages = pythonPackages; in
 let
   pythonPackages = getAttr _pythonPackages pkgs;
 
   pip2nix = import ./nix-common/pip2nix.nix {
     inherit
       pkgs
       pythonPackages;
   };
 
 in
 
 pkgs.stdenv.mkDerivation {
   name = "pip2nix-generated";
   buildInputs = [
     pip2nix.pip2nix
     pythonPackages.pip-tools
     pkgs.apr
     pkgs.aprutil
+    pkgs.libffi
   ];
 
   shellHook = ''
     runHook preShellHook
     echo "Setting SVN_* variables"
     export SVN_LIBRARY_PATH=${pkgs.subversion}/lib
     export SVN_HEADER_PATH=${pkgs.subversion.dev}/include
     runHook postShellHook
   '';
 
   preShellHook = ''
     echo "Starting Generate Shell"
     # Custom prompt to distinguish from other dev envs.
     export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
   '';
 }

requirements.txt

@@ -1,43 +1,48 b''
 ## dependencies
 
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
 
-dogpile.cache==0.7.1
+dogpile.cache==0.9.0
 dogpile.core==0.4.1
 decorator==4.1.2
 dulwich==0.13.0
 hgsubversion==1.9.3
-hg-evolve==8.5.1
-mako==1.0.7
-markupsafe==1.1.0
-mercurial==4.9.1
+hg-evolve==9.1.0
+mako==1.1.0
+markupsafe==1.1.1
+mercurial==5.1.1
 msgpack-python==0.5.6
 
 pastedeploy==2.0.1
 pyramid==1.10.4
-pyramid-mako==1.0.2
+pyramid-mako==1.1.0
+pygit2==0.28.2
 
 repoze.lru==0.7
+redis==3.3.11
 simplejson==3.16.0
 subprocess32==3.5.4
 subvertpy==0.10.1
 
 six==1.11.0
 translationstring==1.3
 webob==1.8.5
 zope.deprecation==4.4.0
 zope.interface==4.6.0
 
 ## http servers
 gevent==1.4.0
 greenlet==0.4.15
 gunicorn==19.9.0
-waitress==1.3.0
+waitress==1.3.1
 
 ## debug
 ipdb==0.12.0
 ipython==5.1.0
 
 ## test related requirements
 -r requirements_test.txt
+
+## uncomment to add the debug libraries
+#-r requirements_debug.txt

requirements_pinned.txt

@@ -1,12 +1,18 b''
 # contains not directly required libraries we want to pin the version.
 
-atomicwrites==1.2.1
-attrs==18.2.0
-hupper==1.6.1
-pathlib2==2.3.4
+atomicwrites==1.3.0
+attrs==19.3.0
+contextlib2==0.6.0.post1
+cffi==1.12.3
+hupper==1.9.1
+importlib-metadata==0.23
+packaging==19.2.0
+pathlib2==2.3.5
 pygments==2.4.2
-psutil==5.5.1
-pluggy==0.11.0
+pyparsing==2.4.5
+psutil==5.6.5
+pluggy==0.13.1
 scandir==1.10.0
 setproctitle==1.1.10
 venusian==1.2.0
+wcwidth==0.1.7

requirements_test.txt

@@ -1,16 +1,16 b''
 # test related requirements
-pytest==3.8.2
-py==1.6.0
-pytest-cov==2.6.0
-pytest-sugar==0.9.1
-pytest-runner==4.2.0
-pytest-profiling==1.3.0
-pytest-timeout==1.3.2
+pytest==4.6.5
+py==1.8.0
+pytest-cov==2.7.1
+pytest-sugar==0.9.2
+pytest-runner==5.1.0
+pytest-profiling==1.7.0
+pytest-timeout==1.3.3
 gprof2dot==2017.9.19
 
-mock==1.0.1
+mock==3.0.5
 cov-core==1.15.0
-coverage==4.5.3
+coverage==4.5.4
 
 webtest==2.0.33
 beautifulsoup4==4.6.3

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.17.4
  No newline at end of file
+4.18.0
  No newline at end of file

vcsserver/base.py

@@ -1,94 +1,76 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import sys
 import traceback
 import logging
 import urlparse
 
 from vcsserver.lib.rc_cache import region_meta
 log = logging.getLogger(__name__)
 
 
 class RepoFactory(object):
     """
     Utility to create instances of repository
 
     It provides internal caching of the `repo` object based on
     the :term:`call context`.
     """
     repo_type = None
 
     def __init__(self):
         self._cache_region = region_meta.dogpile_cache_regions['repo_object']
 
     def _create_config(self, path, config):
         config = {}
         return config
 
     def _create_repo(self, wire, create):
         raise NotImplementedError()
 
     def repo(self, wire, create=False):
-        """
-        Get a repository instance for the given path.
-
-        Uses internally the low level beaker API since the decorators introduce
-        significant overhead.
-        """
-        region = self._cache_region
-        context = wire.get('context', None)
-        repo_path = wire.get('path', '')
-        context_uid = '{}'.format(context)
-        cache = wire.get('cache', True)
-        cache_on = context and cache
-
-        @region.conditional_cache_on_arguments(condition=cache_on)
-        def create_new_repo(_repo_type, _repo_path, _context_uid):
-            return self._create_repo(wire, create)
-
-        repo = create_new_repo(self.repo_type, repo_path, context_uid)
-        return repo
+        raise NotImplementedError()
 
 
 def obfuscate_qs(query_string):
     if query_string is None:
         return None
 
     parsed = []
     for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
         if k in ['auth_token', 'api_key']:
             v = "*****"
         parsed.append((k, v))
 
     return '&'.join('{}{}'.format(
         k, '={}'.format(v) if v else '') for k, v in parsed)
 
 
 def raise_from_original(new_type):
     """
     Raise a new exception type with original args and traceback.
     """
     exc_type, exc_value, exc_traceback = sys.exc_info()
     new_exc = new_type(*exc_value.args)
     # store the original traceback into the new exc
     new_exc._org_exc_tb = traceback.format_exc(exc_traceback)
 
     try:
         raise new_exc, None, exc_traceback
     finally:
         del exc_traceback

vcsserver/git.py

@@ -1,752 +1,1177 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
 import collections
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import traceback
 import urllib
 import urllib2
 from functools import wraps
 
 import more_itertools
+import pygit2
+from pygit2 import Repository as LibGit2Repo
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
-from dulwich.repo import Repo as DulwichRepo, Tag
+from dulwich.repo import Repo as DulwichRepo
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
-from vcsserver.utils import safe_str
-from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
+from vcsserver.utils import safe_str, safe_int
+from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 from vcsserver.git_lfs.lib import LFSOidStore
+from vcsserver.vcs_base import RemoteBase
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
+PEELED_REF_MARKER = '^{}'
+
 
 log = logging.getLogger(__name__)
 
 
+def str_to_dulwich(value):
+    """
+    Dulwich 0.10.1a requires `unicode` objects to be passed in.
+    """
+    return value.decode(settings.WIRE_ENCODING)
+
+
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
+
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except (ChecksumMismatch, WrongObjectException, MissingCommitError,
-                ObjectMissing) as e:
-            exc = exceptions.LookupException(e)
-            raise exc(e)
+        except (ChecksumMismatch, WrongObjectException, MissingCommitError, ObjectMissing,) as e:
+            exc = exceptions.LookupException(org_exc=e)
+            raise exc(safe_str(e))
         except (HangupException, UnexpectedCommandError) as e:
-            exc = exceptions.VcsException(e)
-            raise exc(e)
+            exc = exceptions.VcsException(org_exc=e)
+            raise exc(safe_str(e))
         except Exception as e:
             # NOTE(marcink): becuase of how dulwich handles some exceptions
             # (KeyError on empty repos), we cannot track this and catch all
             # exceptions, it's an exceptions from other handlers
             #if not hasattr(e, '_vcs_kind'):
                 #log.exception("Unhandled exception in git remote call")
                 #raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
-
-    TODO: mikhail: please check if we need this wrapper after updating dulwich
-    to 0.12.0 +
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
+class Repository(LibGit2Repo):
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.free()
+
+
 class GitFactory(RepoFactory):
     repo_type = 'git'
 
-    def _create_repo(self, wire, create):
-        repo_path = str_to_dulwich(wire['path'])
-        return Repo(repo_path)
+    def _create_repo(self, wire, create, use_libgit2=False):
+        if use_libgit2:
+            return Repository(wire['path'])
+        else:
+            repo_path = str_to_dulwich(wire['path'])
+            return Repo(repo_path)
+
+    def repo(self, wire, create=False, use_libgit2=False):
+        """
+        Get a repository instance for the given path.
+        """
+        return self._create_repo(wire, create, use_libgit2)
+
+    def repo_libgit2(self, wire):
+        return self.repo(wire, use_libgit2=True)
 
 
-class GitRemote(object):
+class GitRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
-        self.peeled_ref_marker = '^{}'
         self._bulk_methods = {
-            "author": self.commit_attribute,
-            "date": self.get_object_attrs,
-            "message": self.commit_attribute,
-            "parents": self.commit_attribute,
+            "date": self.date,
+            "author": self.author,
+            "branch": self.branch,
+            "message": self.message,
+            "parents": self.parents,
             "_commit": self.revision,
         }
 
     def _wire_to_config(self, wire):
         if 'config' in wire:
             return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
         return {}
 
-    def _assign_ref(self, wire, ref, commit_id):
-        repo = self._factory.repo(wire)
-        repo[ref] = commit_id
-
     def _remote_conf(self, config):
         params = [
             '-c', 'core.askpass=""',
         ]
         ssl_cert_dir = config.get('vcs_ssl_dir')
         if ssl_cert_dir:
             params.extend(['-c', 'http.sslCAinfo={}'.format(ssl_cert_dir)])
         return params
 
     @reraise_safe_exceptions
-    def is_empty(self, wire):
-        repo = self._factory.repo(wire)
-        try:
-            return not repo.head()
-        except Exception:
-            log.exception("failed to read object_store")
-            return True
+    def discover_git_version(self):
+        stdout, _ = self.run_git_command(
+            {}, ['--version'], _bare=True, _safe=True)
+        prefix = 'git version'
+        if stdout.startswith(prefix):
+            stdout = stdout[len(prefix):]
+        return stdout.strip()
 
     @reraise_safe_exceptions
-    def add_object(self, wire, content):
-        repo = self._factory.repo(wire)
-        blob = objects.Blob()
-        blob.set_raw_string(content)
-        repo.object_store.add_object(blob)
-        return blob.id
+    def is_empty(self, wire):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+
+            try:
+                has_head = repo.head.name
+                if has_head:
+                    return False
+
+                # NOTE(marcink): check again using more expensive method
+                return repo.is_empty
+            except Exception:
+                pass
+
+            return True
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
-        path = wire.get('path')
-        try:
-            self._factory.repo(wire)
-        except NotGitRepository as e:
-            tb = traceback.format_exc()
-            log.debug("Invalid Git path `%s`, tb: %s", path, tb)
-            return False
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _assert_correct_path(_context_uid, _repo_id):
+            try:
+                repo_init = self._factory.repo_libgit2(wire)
+                with repo_init as repo:
+                    pass
+            except pygit2.GitError:
+                path = wire.get('path')
+                tb = traceback.format_exc()
+                log.debug("Invalid Git path `%s`, tb: %s", path, tb)
+                return False
 
-        return True
+            return True
+        return _assert_correct_path(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def bare(self, wire):
-        repo = self._factory.repo(wire)
-        return repo.bare
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            return repo.is_bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
-        repo = self._factory.repo(wire)
-        return repo[sha].as_pretty_string()
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            blob_obj = repo[sha]
+            blob = blob_obj.data
+            return blob
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
-        repo = self._factory.repo(wire)
-        blob = repo[sha]
-        return blob.raw_length()
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _blob_raw_length(_repo_id, _sha):
+
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                blob = repo[sha]
+                return blob.size
+
+        return _blob_raw_length(repo_id, sha)
 
     def _parse_lfs_pointer(self, raw_content):
 
         spec_string = 'version https://git-lfs.github.com/spec'
         if raw_content and raw_content.startswith(spec_string):
             pattern = re.compile(r"""
             (?:\n)?
             ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
             ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
             ^size[ ](?P<oid_size>[0-9]+)\n
             (?:\n)?
             """, re.VERBOSE | re.MULTILINE)
             match = pattern.match(raw_content)
             if match:
                 return match.groupdict()
 
         return {}
 
     @reraise_safe_exceptions
-    def is_large_file(self, wire, sha):
-        repo = self._factory.repo(wire)
-        blob = repo[sha]
-        return self._parse_lfs_pointer(blob.as_raw_string())
+    def is_large_file(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _is_large_file(_repo_id, _sha):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                blob = repo[commit_id]
+                if blob.is_binary:
+                    return {}
+
+                return self._parse_lfs_pointer(blob.data)
+
+        return _is_large_file(repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def is_binary(self, wire, tree_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _is_binary(_repo_id, _tree_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                blob_obj = repo[tree_id]
+                return blob_obj.is_binary
+
+        return _is_binary(repo_id, tree_id)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, oid):
-        repo = self._factory.repo(wire)
         conf = self._wire_to_config(wire)
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
-            repo_name = repo.path
+
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.has_oid()
 
         return False
 
     @reraise_safe_exceptions
     def store_path(self, wire, oid):
-        repo = self._factory.repo(wire)
         conf = self._wire_to_config(wire)
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo_name = repo.path
 
         store_location = conf.get('vcs_git_lfs_store_location')
         if store_location:
-            repo_name = repo.path
             store = LFSOidStore(
                 oid=oid, repo=repo_name, store_location=store_location)
             return store.oid_path
         raise ValueError('Unable to fetch oid with path {}'.format(oid))
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
-        result = {}
-        for attr in pre_load:
-            try:
-                method = self._bulk_methods[attr]
-                args = [wire, rev]
-                if attr == "date":
-                    args.extend(["commit_time", "commit_timezone"])
-                elif attr in ["author", "message", "parents"]:
-                    args.append(attr)
-                result[attr] = method(*args)
-            except KeyError as e:
-                raise exceptions.VcsException(e)(
-                    "Unknown bulk attribute: %s" % attr)
-        return result
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_request(_repo_id, _rev, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_methods[attr]
+                    args = [wire, rev]
+                    result[attr] = method(*args)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(
+                        "Unknown bulk attribute: %s" % attr)
+            return result
+
+        return _bulk_request(repo_id, rev, sorted(pre_load))
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
+    def _type_id_to_name(self, type_id):
+        return {
+            1: b'commit',
+            2: b'tree',
+            3: b'blob',
+            4: b'tag'
+        }[type_id]
+
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError()(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         # TODO(marcink): deprecate this method. Last i checked we don't use it anymore
         remote_refs = self.pull(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
+    @reraise_safe_exceptions
+    def branch(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _branch(_context_uid, _repo_id, _commit_id):
+            regex = re.compile('^refs/heads')
+
+            def filter_with(ref):
+                return regex.match(ref[0]) and ref[1] == _commit_id
+
+            branches = filter(filter_with, self.get_refs(wire).items())
+            return [x[0].split('refs/heads/')[-1] for x in branches]
+
+        return _branch(context_uid, repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def commit_branches(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _commit_branches(_context_uid, _repo_id, _commit_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                branches = [x for x in repo.branches.with_commit(_commit_id)]
+                return branches
+
+        return _commit_branches(context_uid, repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def add_object(self, wire, content):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            blob = objects.Blob()
+            blob.set_raw_string(content)
+            repo.object_store.add_object(blob)
+            return blob.id
+
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
         commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
-                parent.add(
-                    name=node['node_path'], mode=node['mode'], hexsha=blob.id)
+                parent.add(name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.iteritems():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
+        self.create_branch(wire, branch, commit.id)
+
+        # dulwich set-ref
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(e)(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             for k in remote_refs:
-                if k.endswith(self.peeled_ref_marker):
+                if k.endswith(PEELED_REF_MARKER):
                     log.debug("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs and not update_after:
                 # mikhail: explicitly set the head to the last ref.
                 repo['HEAD'] = remote_refs[refs[-1]]
 
         if update_after:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
         return remote_refs
 
     @reraise_safe_exceptions
-    def sync_fetch(self, wire, url, refs=None):
+    def sync_fetch(self, wire, url, refs=None, all_refs=False):
         repo = self._factory.repo(wire)
         if refs and not isinstance(refs, (list, tuple)):
             refs = [refs]
+
         config = self._wire_to_config(wire)
         # get all remote refs we'll use to fetch later
+        cmd = ['ls-remote']
+        if not all_refs:
+            cmd += ['--heads', '--tags']
+        cmd += [url]
         output, __ = self.run_git_command(
-            wire, ['ls-remote', url], fail_on_stderr=False,
+            wire, cmd, fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         remote_refs = collections.OrderedDict()
         fetch_refs = []
 
         for ref_line in output.splitlines():
             sha, ref = ref_line.split('\t')
             sha = sha.strip()
             if ref in remote_refs:
                 # duplicate, skip
                 continue
-            if ref.endswith(self.peeled_ref_marker):
+            if ref.endswith(PEELED_REF_MARKER):
                 log.debug("Skipping peeled reference %s", ref)
                 continue
             # don't sync HEAD
             if ref in ['HEAD']:
                 continue
 
             remote_refs[ref] = sha
 
             if refs and sha in refs:
                 # we filter fetch using our specified refs
                 fetch_refs.append('{}:{}'.format(ref, ref))
             elif not refs:
                 fetch_refs.append('{}:{}'.format(ref, ref))
         log.debug('Finished obtaining fetch refs, total: %s', len(fetch_refs))
+
         if fetch_refs:
             for chunk in more_itertools.chunked(fetch_refs, 1024 * 4):
                 fetch_refs_chunks = list(chunk)
                 log.debug('Fetching %s refs from import url', len(fetch_refs_chunks))
                 _out, _err = self.run_git_command(
                     wire, ['fetch', url, '--force', '--prune', '--'] + fetch_refs_chunks,
                     fail_on_stderr=False,
                     _copts=self._remote_conf(config),
                     extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
         return remote_refs
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url, refs=None):
         if not self.check_url(url, wire):
             return
         config = self._wire_to_config(wire)
-        repo = self._factory.repo(wire)
+        self._factory.repo(wire)
         self.run_git_command(
             wire, ['push', url, '--mirror'], fail_on_stderr=False,
             _copts=self._remote_conf(config),
             extra_env={'GIT_TERMINAL_PROMPT': '0'})
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha):
-        repo = self._factory.repo(wire)
-        obj = repo.get_object(sha)
-        commit_id = obj.id
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_object(_context_uid, _repo_id, _sha):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
 
-        if isinstance(obj, Tag):
-            commit_id = obj.object[1]
+                missing_commit_err = 'Commit {} does not exist for `{}`'.format(sha, wire['path'])
+                try:
+                    commit = repo.revparse_single(sha)
+                except (KeyError, ValueError) as e:
+                    raise exceptions.LookupException(e)(missing_commit_err)
+
+                is_tag = False
+                if isinstance(commit, pygit2.Tag):
+                    commit = repo.get(commit.target)
+                    is_tag = True
+
+                check_dangling = True
+                if is_tag:
+                    check_dangling = False
 
-        return {
-            'id': obj.id,
-            'type': obj.type_name,
-            'commit_id': commit_id,
-            'idx': 0
-        }
+                # we used a reference and it parsed means we're not having a dangling commit
+                if sha != commit.hex:
+                    check_dangling = False
+
+                if check_dangling:
+                    # check for dangling commit
+                    for branch in repo.branches.with_commit(commit.hex):
+                        if branch:
+                            break
+                    else:
+                        raise exceptions.LookupException(None)(missing_commit_err)
 
-    @reraise_safe_exceptions
-    def get_object_attrs(self, wire, sha, *attrs):
-        repo = self._factory.repo(wire)
-        obj = repo.get_object(sha)
-        return list(getattr(obj, a) for a in attrs)
+                commit_id = commit.hex
+                type_id = commit.type
+
+                return {
+                    'id': commit_id,
+                    'type': self._type_id_to_name(type_id),
+                    'commit_id': commit_id,
+                    'idx': 0
+                }
+
+        return _get_object(context_uid, repo_id, sha)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
-        repo = self._factory.repo(wire)
-        result = {}
-        for ref, sha in repo.refs.as_dict().items():
-            peeled_sha = repo.get_peeled(ref)
-            result[ref] = peeled_sha
-        return result
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_refs(_context_uid, _repo_id):
+
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                regex = re.compile('^refs/(heads|tags)/')
+                return {x.name: x.target.hex for x in
+                        filter(lambda ref: regex.match(ref.name) ,repo.listall_reference_objects())}
+
+        return _get_refs(context_uid, repo_id)
 
     @reraise_safe_exceptions
-    def get_refs_path(self, wire):
-        repo = self._factory.repo(wire)
-        return repo.refs.path
+    def get_branch_pointers(self, wire):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_branch_pointers(_context_uid, _repo_id):
+
+            repo_init = self._factory.repo_libgit2(wire)
+            regex = re.compile('^refs/heads')
+            with repo_init as repo:
+                branches = filter(lambda ref: regex.match(ref.name), repo.listall_reference_objects())
+                return {x.target.hex: x.shorthand for x in branches}
+
+        return _get_branch_pointers(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def head(self, wire, show_exc=True):
-        repo = self._factory.repo(wire)
-        try:
-            return repo.head()
-        except Exception:
-            if show_exc:
-                raise
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _head(_context_uid, _repo_id, _show_exc):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                try:
+                    return repo.head.peel().hex
+                except Exception:
+                    if show_exc:
+                        raise
+        return _head(context_uid, repo_id, show_exc)
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
-        repo = self._factory.repo(wire)
-        obj = repo[rev]
-        obj_data = {
-            'id': obj.id,
-        }
-        try:
-            obj_data['tree'] = obj.tree
-        except AttributeError:
-            pass
-        return obj_data
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _revision(_context_uid, _repo_id, _rev):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[rev]
+                obj_data = {
+                    'id': commit.id.hex,
+                }
+                # tree objects itself don't have tree_id attribute
+                if hasattr(commit, 'tree_id'):
+                    obj_data['tree'] = commit.tree_id.hex
+
+                return obj_data
+        return _revision(context_uid, repo_id, rev)
+
+    @reraise_safe_exceptions
+    def date(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _date(_repo_id, _commit_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[commit_id]
+
+                if hasattr(commit, 'commit_time'):
+                    commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
+                else:
+                    commit = commit.get_object()
+                    commit_time, commit_time_offset = commit.commit_time, commit.commit_time_offset
+
+                # TODO(marcink): check dulwich difference of offset vs timezone
+                return [commit_time, commit_time_offset]
+        return _date(repo_id, commit_id)
 
     @reraise_safe_exceptions
-    def commit_attribute(self, wire, rev, attr):
-        repo = self._factory.repo(wire)
-        obj = repo[rev]
-        return getattr(obj, attr)
+    def author(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _author(_repo_id, _commit_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[commit_id]
+
+                if hasattr(commit, 'author'):
+                    author = commit.author
+                else:
+                    author = commit.get_object().author
+
+                if author.email:
+                    return u"{} <{}>".format(author.name, author.email)
+
+                return u"{}".format(author.raw_name)
+        return _author(repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def message(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _message(_repo_id, _commit_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[commit_id]
+                return commit.message
+        return _message(repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def parents(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _parents(_repo_id, _commit_id):
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                commit = repo[commit_id]
+                if hasattr(commit, 'parent_ids'):
+                    parent_ids = commit.parent_ids
+                else:
+                    parent_ids = commit.get_object().parent_ids
+
+                return [x.hex for x in parent_ids]
+        return _parents(repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def children(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _children(_repo_id, _commit_id):
+            output, __ = self.run_git_command(
+                wire, ['rev-list', '--all', '--children'])
+
+            child_ids = []
+            pat = re.compile(r'^%s' % commit_id)
+            for l in output.splitlines():
+                if pat.match(l):
+                    found_ids = l.split(' ')[1:]
+                    child_ids.extend(found_ids)
+
+            return child_ids
+        return _children(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
-        repo = self._factory.repo(wire)
-        repo.refs[key] = value
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo.references.create(key, value, force=True)
+
+    @reraise_safe_exceptions
+    def create_branch(self, wire, branch_name, commit_id, force=False):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            commit = repo[commit_id]
+
+            if force:
+                repo.branches.local.create(branch_name, commit, force=force)
+            elif not repo.branches.get(branch_name):
+                # create only if that branch isn't existing
+                repo.branches.local.create(branch_name, commit, force=force)
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
-        repo = self._factory.repo(wire)
-        del repo.refs[key]
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            repo.references.delete(key)
+
+    @reraise_safe_exceptions
+    def tag_remove(self, wire, tag_name):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            key = 'refs/tags/{}'.format(tag_name)
+            repo.references.delete(key)
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
+        # TODO(marcink): remove this seems it's only used by tests
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
+    def tree_and_type_for_path(self, wire, commit_id, path):
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
+            repo_init = self._factory.repo_libgit2(wire)
+
+            with repo_init as repo:
+                commit = repo[commit_id]
+                try:
+                    tree = commit.tree[path]
+                except KeyError:
+                    return None, None, None
+
+                return tree.id.hex, tree.type, tree.filemode
+        return _tree_and_type_for_path(context_uid, repo_id, commit_id, path)
+
+    @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
-        repo = self._factory.repo(wire)
-        tree = repo[tree_id]
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _tree_items(_repo_id, _tree_id):
+
+            repo_init = self._factory.repo_libgit2(wire)
+            with repo_init as repo:
+                try:
+                    tree = repo[tree_id]
+                except KeyError:
+                    raise ObjectMissing('No tree with id: {}'.format(tree_id))
+
+                result = []
+                for item in tree:
+                    item_sha = item.hex
+                    item_mode = item.filemode
+                    item_type = item.type
+
+                    if item_type == 'commit':
+                        # NOTE(marcink): submodules we translate to 'link' for backward compat
+                        item_type = 'link'
+
+                    result.append((item.name, item_mode, item_sha, item_type))
+                return result
+        return _tree_items(repo_id, tree_id)
+
+    @reraise_safe_exceptions
+    def diff_2(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
+        """
+        Old version that uses subprocess to call diff
+        """
+
+        flags = [
+            '-U%s' % context, '--patch',
+            '--binary',
+            '--find-renames',
+            '--no-indent-heuristic',
+            # '--indent-heuristic',
+            #'--full-index',
+            #'--abbrev=40'
+        ]
+
+        if opt_ignorews:
+            flags.append('--ignore-all-space')
+
+        if commit_id_1 == self.EMPTY_COMMIT:
+            cmd = ['show'] + flags + [commit_id_2]
+        else:
+            cmd = ['diff'] + flags + [commit_id_1, commit_id_2]
+
+        if file_filter:
+            cmd.extend(['--', file_filter])
+
+        diff, __ = self.run_git_command(wire, cmd)
+        # If we used 'show' command, strip first few lines (until actual diff
+        # starts)
+        if commit_id_1 == self.EMPTY_COMMIT:
+            lines = diff.splitlines()
+            x = 0
+            for line in lines:
+                if line.startswith('diff'):
+                    break
+                x += 1
+            # Append new line just like 'diff' command do
+            diff = '\n'.join(lines[x:]) + '\n'
+        return diff
+
+    @reraise_safe_exceptions
+    def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_ignorews, context):
+        repo_init = self._factory.repo_libgit2(wire)
+        with repo_init as repo:
+            swap = True
+            flags = 0
+            flags |= pygit2.GIT_DIFF_SHOW_BINARY
+
+            if opt_ignorews:
+                flags |= pygit2.GIT_DIFF_IGNORE_WHITESPACE
+
+            if commit_id_1 == self.EMPTY_COMMIT:
+                comm1 = repo[commit_id_2]
+                diff_obj = comm1.tree.diff_to_tree(
+                    flags=flags, context_lines=context, swap=swap)
+
+            else:
+                comm1 = repo[commit_id_2]
+                comm2 = repo[commit_id_1]
+                diff_obj = comm1.tree.diff_to_tree(
+                    comm2.tree, flags=flags, context_lines=context, swap=swap)
+            similar_flags = 0
+            similar_flags |= pygit2.GIT_DIFF_FIND_RENAMES
+            diff_obj.find_similar(flags=similar_flags)
+
+            if file_filter:
+                for p in diff_obj:
+                    if p.delta.old_file.path == file_filter:
+                        return p.patch or ''
+                # fo matching path == no diff
+                return ''
+            return diff_obj.patch or ''
+
+    @reraise_safe_exceptions
+    def node_history(self, wire, commit_id, path, limit):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
+            # optimize for n==1, rev-list is much faster for that use-case
+            if limit == 1:
+                cmd = ['rev-list', '-1', commit_id, '--', path]
+            else:
+                cmd = ['log']
+                if limit:
+                    cmd.extend(['-n', str(safe_int(limit, 0))])
+                cmd.extend(['--pretty=format: %H', '-s', commit_id, '--', path])
+
+            output, __ = self.run_git_command(wire, cmd)
+            commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
+
+            return [x for x in commit_ids]
+        return _node_history(context_uid, repo_id, commit_id, path, limit)
+
+    @reraise_safe_exceptions
+    def node_annotate(self, wire, commit_id, path):
+
+        cmd = ['blame', '-l', '--root', '-r', commit_id, '--', path]
+        # -l     ==> outputs long shas (and we need all 40 characters)
+        # --root ==> doesn't put '^' character for boundaries
+        # -r commit_id ==> blames for the given commit
+        output, __ = self.run_git_command(wire, cmd)
 
         result = []
-        for item in tree.iteritems():
-            item_sha = item.sha
-            item_mode = item.mode
-
-            if FILE_MODE(item_mode) == GIT_LINK:
-                item_type = "link"
-            else:
-                item_type = repo[item_sha].type_name
-
-            result.append((item.path, item_mode, item_sha, item_type))
+        for i, blame_line in enumerate(output.split('\n')[:-1]):
+            line_no = i + 1
+            commit_id, line = re.split(r' ', blame_line, 1)
+            result.append((line_no, commit_id, line))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
-    def discover_git_version(self):
-        stdout, _ = self.run_git_command(
-            {}, ['--version'], _bare=True, _safe=True)
-        prefix = 'git version'
-        if stdout.startswith(prefix):
-            stdout = stdout[len(prefix):]
-        return stdout.strip()
+    def get_all_commit_ids(self, wire):
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_all_commit_ids(_context_uid, _repo_id):
+
+            cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
+            try:
+                output, __ = self.run_git_command(wire, cmd)
+                return output.splitlines()
+            except Exception:
+                # Can be raised for empty repositories
+                return []
+        return _get_all_commit_ids(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         if '_copts' in opts:
             _copts.extend(opts['_copts'] or [])
             del opts['_copts']
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
         _opts = {'env': gitenv, 'shell': False}
 
+        proc = None
         try:
             _opts.update(opts)
-            p = subprocessio.SubprocessIOChunker(cmd, **_opts)
+            proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
-            return ''.join(p), ''.join(p.error)
+            return ''.join(proc), ''.join(proc.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
+        finally:
+            if proc:
+                proc.close()
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_git_hooks
-        repo = self._factory.repo(wire)
-        return install_git_hooks(repo.path, repo.bare, force_create=force)
+        bare = self.bare(wire)
+        path = wire['path']
+        return install_git_hooks(path, bare, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_git_pre_hook_version, get_git_post_hook_version)
-        repo = self._factory.repo(wire)
+        bare = self.bare(wire)
+        path = wire['path']
         return {
-            'pre_version': get_git_pre_hook_version(repo.path, repo.bare),
-            'post_version': get_git_post_hook_version(repo.path, repo.bare),
+            'pre_version': get_git_pre_hook_version(path, bare),
+            'post_version': get_git_post_hook_version(path, bare),
         }
-
-
-def str_to_dulwich(value):
-    """
-    Dulwich 0.10.1a requires `unicode` objects to be passed in.
-    """
-    return value.decode(settings.WIRE_ENCODING)

vcsserver/hg.py

@@ -1,856 +1,990 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import logging
 import stat
 import urllib
 import urllib2
 import traceback
 
-from hgext import largefiles, rebase
+from hgext import largefiles, rebase, purge
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
 from mercurial import verify
 
 import vcsserver
 from vcsserver import exceptions
 from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex, get_ctx,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     makepeer, instance, match, memctx, exchange, memfilectx, nullrev, hg_merge,
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError)
+from vcsserver.vcs_base import RemoteBase
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
 
     class LoggingUI(ui.ui):
         def status(self, *msg, **opts):
             log.info(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).status(*msg, **opts)
 
         def warn(self, *msg, **opts):
             log.warn(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).warn(*msg, **opts)
 
         def error(self, *msg, **opts):
             log.error(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).error(*msg, **opts)
 
         def note(self, *msg, **opts):
             log.info(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).note(*msg, **opts)
 
         def debug(self, *msg, **opts):
             log.debug(' '.join(msg).rstrip('\n'))
             super(LoggingUI, self).debug(*msg, **opts)
 
     baseui = LoggingUI()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     baseui.setconfig('ui', 'paginate', 'never')
     # for better Error reporting of Mercurial
     baseui.setconfig('ui', 'message-output', 'stderr')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig('worker', 'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig('extensions', 'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig('extensions', 'largefiles', '!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
+
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired) as e:
             raise_from_original(exceptions.AbortException(e))
         except RepoLookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except RequirementError as e:
             raise_from_original(exceptions.RequirementException(e))
         except RepoError as e:
             raise_from_original(exceptions.VcsException(e))
         except LookupError as e:
             raise_from_original(exceptions.LookupException(e))
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException(e))
 
             raise
     return wrapper
 
 
 class MercurialFactory(RepoFactory):
     repo_type = 'hg'
 
     def _create_config(self, config, hooks=True):
         if not hooks:
             hooks_to_clean = frozenset((
                 'changegroup.repo_size', 'preoutgoing.pre_pull',
                 'outgoing.pull_logger', 'prechangegroup.pre_push'))
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         return instance(baseui, wire["path"], create)
 
+    def repo(self, wire, create=False):
+        """
+        Get a repository instance for the given path.
+        """
+        return self._create_repo(wire, create)
 
-class HgRemote(object):
+
+class HgRemote(RemoteBase):
 
     def __init__(self, factory):
         self._factory = factory
-
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "obsolete": self.ctx_obsolete,
             "phase": self.ctx_phase,
             "hidden": self.ctx_hidden,
             "_file_paths": self.ctx_list,
         }
 
     def _get_ctx(self, repo, ref):
         return get_ctx(repo, ref)
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return util.version()
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
         repo = self._factory.repo(wire)
 
         try:
             return len(repo) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
     @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
         elif kind == "tbz2":
             archiver = archival.tarit(archive_path, mtime, "bz2")
         elif kind == 'zip':
             archiver = archival.zipit(archive_path, mtime)
         else:
             raise exceptions.ArchiveException()(
                 'Remote does not support: "%s".' % kind)
 
         for f_path, f_mode, f_is_link, f_content in file_info:
             archiver.addfile(f_path, f_mode, f_is_link, f_content)
         archiver.done()
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
-        repo = self._factory.repo(wire)
-        return dict(repo._bookmarks)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _bookmarks(_context_uid, _repo_id):
+            repo = self._factory.repo(wire)
+            return dict(repo._bookmarks)
+
+        return _bookmarks(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
-        repo = self._factory.repo(wire)
-        iter_branches = repo.branchmap().iterbranches()
-        bt = {}
-        for branch_name, _heads, tip, is_closed in iter_branches:
-            if normal and not is_closed:
-                bt[branch_name] = tip
-            if closed and is_closed:
-                bt[branch_name] = tip
-
-        return bt
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _branches(_context_uid, _repo_id, _normal, _closed):
+            repo = self._factory.repo(wire)
+            iter_branches = repo.branchmap().iterbranches()
+            bt = {}
+            for branch_name, _heads, tip, is_closed in iter_branches:
+                if normal and not is_closed:
+                    bt[branch_name] = tip
+                if closed and is_closed:
+                    bt[branch_name] = tip
 
-    @reraise_safe_exceptions
-    def bulk_request(self, wire, rev, pre_load):
-        result = {}
-        for attr in pre_load:
-            try:
-                method = self._bulk_methods[attr]
-                result[attr] = method(wire, rev)
-            except KeyError as e:
-                raise exceptions.VcsException(e)(
-                    'Unknown bulk attribute: "%s"' % attr)
-        return result
+            return bt
 
-    @reraise_safe_exceptions
-    def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
-        baseui = self._factory._create_config(wire["config"], hooks=hooks)
-        clone(baseui, source, dest, noupdate=not update_after_clone)
+        return _branches(context_uid, repo_id, normal, closed)
 
     @reraise_safe_exceptions
-    def commitctx(
-            self, wire, message, parents, commit_time, commit_timezone,
-            user, files, extra, removed, updated):
-
-        repo = self._factory.repo(wire)
-        baseui = self._factory._create_config(wire['config'])
-        publishing = baseui.configbool('phases', 'publish')
-        if publishing:
-            new_commit = 'public'
-        else:
-            new_commit = 'draft'
-
-        def _filectxfn(_repo, ctx, path):
-            """
-            Marks given path as added/changed/removed in a given _repo. This is
-            for internal mercurial commit function.
-            """
-
-            # check if this path is removed
-            if path in removed:
-                # returning None is a way to mark node for removal
-                return None
+    def bulk_request(self, wire, commit_id, pre_load):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _bulk_request(_repo_id, _commit_id, _pre_load):
+            result = {}
+            for attr in pre_load:
+                try:
+                    method = self._bulk_methods[attr]
+                    result[attr] = method(wire, commit_id)
+                except KeyError as e:
+                    raise exceptions.VcsException(e)(
+                        'Unknown bulk attribute: "%s"' % attr)
+            return result
 
-            # check if this path is added
-            for node in updated:
-                if node['path'] == path:
-                    return memfilectx(
-                        _repo,
-                        changectx=ctx,
-                        path=node['path'],
-                        data=node['content'],
-                        islink=False,
-                        isexec=bool(node['mode'] & stat.S_IXUSR),
-                        copied=False)
-
-            raise exceptions.AbortException()(
-                "Given path haven't been marked as added, "
-                "changed or removed (%s)" % path)
-
-        with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
-
-            commit_ctx = memctx(
-                repo=repo,
-                parents=parents,
-                text=message,
-                files=files,
-                filectxfn=_filectxfn,
-                user=user,
-                date=(commit_time, commit_timezone),
-                extra=extra)
-
-            n = repo.commitctx(commit_ctx)
-            new_id = hex(n)
-
-            return new_id
+        return _bulk_request(repo_id, commit_id, sorted(pre_load))
 
     @reraise_safe_exceptions
-    def ctx_branch(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return ctx.branch()
+    def ctx_branch(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_branch(_repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return ctx.branch()
+        return _ctx_branch(repo_id, commit_id)
 
     @reraise_safe_exceptions
-    def ctx_children(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return [child.rev() for child in ctx.children()]
-
-    @reraise_safe_exceptions
-    def ctx_date(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return ctx.date()
+    def ctx_date(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_date(_repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return ctx.date()
+        return _ctx_date(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.description()
 
     @reraise_safe_exceptions
-    def ctx_files(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return ctx.files()
+    def ctx_files(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_files(_repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return ctx.files()
+
+        return _ctx_files(repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = self._get_ctx(repo, revision)
         return list(ctx)
 
     @reraise_safe_exceptions
-    def ctx_parents(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return [parent.rev() for parent in ctx.parents()]
+    def ctx_parents(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_parents(_repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return [parent.hex() for parent in ctx.parents()
+                    if not (parent.hidden() or parent.obsolete())]
+
+        return _ctx_parents(repo_id, commit_id)
+
+    @reraise_safe_exceptions
+    def ctx_children(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_children(_repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return [child.hex() for child in ctx.children()
+                    if not (child.hidden() or child.obsolete())]
+
+        return _ctx_children(repo_id, commit_id)
 
     @reraise_safe_exceptions
-    def ctx_phase(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        # public=0, draft=1, secret=3
-        return ctx.phase()
+    def ctx_phase(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_phase(_context_uid, _repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            # public=0, draft=1, secret=3
+            return ctx.phase()
+        return _ctx_phase(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
-    def ctx_obsolete(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return ctx.obsolete()
+    def ctx_obsolete(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return ctx.obsolete()
+        return _ctx_obsolete(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
-    def ctx_hidden(self, wire, revision):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        return ctx.hidden()
+    def ctx_hidden(self, wire, commit_id):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _ctx_hidden(_context_uid, _repo_id, _commit_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            return ctx.hidden()
+        return _ctx_hidden(context_uid, repo_id, commit_id)
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
 
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError()('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError(e)("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
                     "Verifying if URL is a Mercurial repository: %s",
                     cleaned_uri)
                 ui = make_ui_from_config(config)
                 peer_checker = makepeer(ui, url)
                 peer_checker.lookup('tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
                         cleaned_uri)
             raise exceptions.URLError(e)(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
         log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
-    def diff(
-            self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
-            context):
+    def diff(self, wire, commit_id_1, commit_id_2, file_filter, opt_git, opt_ignorews, context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             match_filter = match(file_filter[0], '', [file_filter[1]])
         else:
             match_filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
 
         try:
             return "".join(patch.diff(
-                repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
+                repo, node1=commit_id_1, node2=commit_id_2, match=match_filter, opts=opts))
         except RepoLookupError as e:
             raise exceptions.LookupException(e)()
 
     @reraise_safe_exceptions
     def node_history(self, wire, revision, path, limit):
-        repo = self._factory.repo(wire)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
+            repo = self._factory.repo(wire)
 
-        ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
+            ctx = self._get_ctx(repo, revision)
+            fctx = ctx.filectx(path)
 
-        def history_iter():
-            limit_rev = fctx.rev()
-            for obj in reversed(list(fctx.filelog())):
-                obj = fctx.filectx(obj)
-                ctx = obj.changectx()
-                if ctx.hidden() or ctx.obsolete():
-                    continue
+            def history_iter():
+                limit_rev = fctx.rev()
+                for obj in reversed(list(fctx.filelog())):
+                    obj = fctx.filectx(obj)
+                    ctx = obj.changectx()
+                    if ctx.hidden() or ctx.obsolete():
+                        continue
 
-                if limit_rev >= obj.rev():
-                    yield obj
+                    if limit_rev >= obj.rev():
+                        yield obj
 
-        history = []
-        for cnt, obj in enumerate(history_iter()):
-            if limit and cnt >= limit:
-                break
-            history.append(hex(obj.node()))
+            history = []
+            for cnt, obj in enumerate(history_iter()):
+                if limit and cnt >= limit:
+                    break
+                history.append(hex(obj.node()))
 
-        return [x for x in history]
+            return [x for x in history]
+        return _node_history(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
     def node_history_untill(self, wire, revision, path, limit):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _node_history_until(_context_uid, _repo_id):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, revision)
+            fctx = ctx.filectx(path)
 
-        file_log = list(fctx.filelog())
-        if limit:
-            # Limit to the last n items
-            file_log = file_log[-limit:]
+            file_log = list(fctx.filelog())
+            if limit:
+                # Limit to the last n items
+                file_log = file_log[-limit:]
 
-        return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
+            return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
+        return _node_history_until(context_uid, repo_id, revision, path, limit)
 
     @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(path)
 
         result = []
         for i, annotate_obj in enumerate(fctx.annotate(), 1):
             ln_no = i
             sha = hex(annotate_obj.fctx.node())
             content = annotate_obj.text
             result.append((ln_no, sha, content))
         return result
 
     @reraise_safe_exceptions
-    def fctx_data(self, wire, revision, path):
+    def fctx_node_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, revision)
         fctx = ctx.filectx(path)
         return fctx.data()
 
     @reraise_safe_exceptions
-    def fctx_flags(self, wire, revision, path):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
-        return fctx.flags()
+    def fctx_flags(self, wire, commit_id, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _fctx_flags(_repo_id, _commit_id, _path):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            fctx = ctx.filectx(path)
+            return fctx.flags()
+
+        return _fctx_flags(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
-    def fctx_size(self, wire, revision, path):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        fctx = ctx.filectx(path)
-        return fctx.size()
+    def fctx_size(self, wire, commit_id, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _fctx_size(_repo_id, _revision, _path):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, commit_id)
+            fctx = ctx.filectx(path)
+            return fctx.size()
+        return _fctx_size(repo_id, commit_id, path)
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
-        repo = self._factory.repo(wire)
-        repo = repo.filtered(name)
-        revs = map(lambda x: hex(x[7]), repo.changelog.index)
-        return revs
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_all_commit_ids(_context_uid, _repo_id, _name):
+            repo = self._factory.repo(wire)
+            repo = repo.filtered(name)
+            revs = map(lambda x: hex(x[7]), repo.changelog.index)
+            return revs
+        return _get_all_commit_ids(context_uid, repo_id, name)
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
-    def get_config_bool(self, wire, section, name, untrusted=False):
-        repo = self._factory.repo(wire)
-        return repo.ui.configbool(section, name, untrusted=untrusted)
+    def is_large_file(self, wire, commit_id, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
+            return largefiles.lfutil.isstandin(path)
+
+        return _is_large_file(context_uid, repo_id, commit_id, path)
 
     @reraise_safe_exceptions
-    def get_config_list(self, wire, section, name, untrusted=False):
-        repo = self._factory.repo(wire)
-        return repo.ui.configlist(section, name, untrusted=untrusted)
+    def is_binary(self, wire, revision, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
 
-    @reraise_safe_exceptions
-    def is_large_file(self, wire, path):
-        return largefiles.lfutil.isstandin(path)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _is_binary(_repo_id, _sha, _path):
+            repo = self._factory.repo(wire)
+            ctx = self._get_ctx(repo, revision)
+            fctx = ctx.filectx(path)
+            return fctx.isbinary()
+
+        return _is_binary(repo_id, revision, path)
 
     @reraise_safe_exceptions
     def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
-
-        repo = self._factory.repo(wire)
-
-        if isinstance(revision, int):
-            # NOTE(marcink):
-            # since Mercurial doesn't support negative indexes properly
-            # we need to shift accordingly by one to get proper index, e.g
-            # repo[-1] => repo[-2]
-            # repo[0]  => repo[-1]
-            if revision <= 0:
-                revision = revision + -1
-        try:
-            ctx = self._get_ctx(repo, revision)
-        except (TypeError, RepoLookupError) as e:
-            e._org_exc_tb = traceback.format_exc()
-            raise exceptions.LookupException(e)(revision)
-        except LookupError as e:
-            e._org_exc_tb = traceback.format_exc()
-            raise exceptions.LookupException(e)(e.name)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _lookup(_context_uid, _repo_id, _revision, _both):
 
-        if not both:
-            return ctx.hex()
-
-        ctx = repo[ctx.hex()]
-        return ctx.hex(), ctx.rev()
+            repo = self._factory.repo(wire)
+            rev = _revision
+            if isinstance(rev, int):
+                # NOTE(marcink):
+                # since Mercurial doesn't support negative indexes properly
+                # we need to shift accordingly by one to get proper index, e.g
+                # repo[-1] => repo[-2]
+                # repo[0]  => repo[-1]
+                if rev <= 0:
+                    rev = rev + -1
+            try:
+                ctx = self._get_ctx(repo, rev)
+            except (TypeError, RepoLookupError) as e:
+                e._org_exc_tb = traceback.format_exc()
+                raise exceptions.LookupException(e)(rev)
+            except LookupError as e:
+                e._org_exc_tb = traceback.format_exc()
+                raise exceptions.LookupException(e)(e.name)
 
-    @reraise_safe_exceptions
-    def pull(self, wire, url, commit_ids=None):
-        repo = self._factory.repo(wire)
-        # Disable any prompts for this repo
-        repo.ui.setconfig('ui', 'interactive', 'off', '-y')
+            if not both:
+                return ctx.hex()
 
-        remote = peer(repo, {}, url)
-        # Disable any prompts for this remote
-        remote.ui.setconfig('ui', 'interactive', 'off', '-y')
+            ctx = repo[ctx.hex()]
+            return ctx.hex(), ctx.rev()
 
-        if commit_ids:
-            commit_ids = [bin(commit_id) for commit_id in commit_ids]
-
-        return exchange.pull(
-            repo, remote, heads=commit_ids, force=None).cgresult
+        return _lookup(context_uid, repo_id, revision, both)
 
     @reraise_safe_exceptions
     def sync_push(self, wire, url):
         if not self.check_url(url, wire['config']):
             return
 
         repo = self._factory.repo(wire)
 
         # Disable any prompts for this repo
         repo.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         bookmarks = dict(repo._bookmarks).keys()
         remote = peer(repo, {}, url)
         # Disable any prompts for this remote
         remote.ui.setconfig('ui', 'interactive', 'off', '-y')
 
         return exchange.push(
             repo, remote, newbranch=True, bookmarks=bookmarks).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = self._get_ctx(repo, rev)
         return ctx.rev()
 
     @reraise_safe_exceptions
-    def rev_range(self, wire, filter):
-        repo = self._factory.repo(wire)
-        revisions = [rev for rev in revrange(repo, filter)]
-        return revisions
+    def rev_range(self, wire, commit_filter):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _rev_range(_context_uid, _repo_id, _filter):
+            repo = self._factory.repo(wire)
+            revisions = [rev for rev in revrange(repo, commit_filter)]
+            return revisions
+
+        return _rev_range(context_uid, repo_id, sorted(commit_filter))
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         other_path = kwargs.pop('other_path', None)
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.makeunionrepository(baseui, other_path, wire["path"])
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
-    def strip(self, wire, revision, update, backup):
-        repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        hgext_strip(
-            repo.baseui, repo, ctx.node(), update=update, backup=backup)
-
-    @reraise_safe_exceptions
     def verify(self, wire,):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         baseui.setconfig('ui', 'quiet', 'false')
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
         baseui.write = write
 
         repo.ui = baseui
         verify.verify(repo)
         return output.getvalue()
 
     @reraise_safe_exceptions
-    def tag(self, wire, name, revision, message, local, user,
-            tag_time, tag_timezone):
+    def hg_update_cache(self, wire,):
         repo = self._factory.repo(wire)
-        ctx = self._get_ctx(repo, revision)
-        node = ctx.node()
+        baseui = self._factory._create_config(wire['config'])
+        baseui.setconfig('ui', 'quiet', 'false')
+        output = io.BytesIO()
 
-        date = (tag_time, tag_timezone)
-        try:
-            hg_tag.tag(repo, name, node, message, local, user, date)
-        except Abort as e:
-            log.exception("Tag operation aborted")
-            # Exception can contain unicode which we convert
-            raise exceptions.AbortException(e)(repr(e))
+        def write(data, **unused_kwargs):
+            output.write(data)
+        baseui.write = write
+
+        repo.ui = baseui
+        with repo.wlock(), repo.lock():
+            repo.updatecaches(full=True)
+
+        return output.getvalue()
 
     @reraise_safe_exceptions
     def tags(self, wire):
-        repo = self._factory.repo(wire)
-        return repo.tags()
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _tags(_context_uid, _repo_id):
+            repo = self._factory.repo(wire)
+            return repo.tags()
+
+        return _tags(context_uid, repo_id)
 
     @reraise_safe_exceptions
     def update(self, wire, node=None, clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
-    def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
-                 hooks=True):
-        repo = self._factory.repo(wire)
-        baseui = self._factory._create_config(wire['config'], hooks=hooks)
-
-        # Mercurial internally has a lot of logic that checks ONLY if
-        # option is defined, we just pass those if they are defined then
-        opts = {}
-        if bookmark:
-            opts['bookmark'] = bookmark
-        if branch:
-            opts['branch'] = branch
-        if revision:
-            opts['rev'] = revision
-
-        commands.pull(baseui, repo, source, **opts)
-
-    @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [branch]
         else:
             args = []
         commands.heads(baseui, repo, template='{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
         changelog = repo.changelog
         lookup = repo.lookup
         a = changelog.ancestor(lookup(revision1), lookup(revision2))
         return hex(a)
 
     @reraise_safe_exceptions
-    def push(self, wire, revisions, dest_path, hooks=True,
-             push_branches=False):
+    def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
+        baseui = self._factory._create_config(wire["config"], hooks=hooks)
+        clone(baseui, source, dest, noupdate=not update_after_clone)
+
+    @reraise_safe_exceptions
+    def commitctx(self, wire, message, parents, commit_time, commit_timezone, user, files, extra, removed, updated):
+
+        repo = self._factory.repo(wire)
+        baseui = self._factory._create_config(wire['config'])
+        publishing = baseui.configbool('phases', 'publish')
+        if publishing:
+            new_commit = 'public'
+        else:
+            new_commit = 'draft'
+
+        def _filectxfn(_repo, ctx, path):
+            """
+            Marks given path as added/changed/removed in a given _repo. This is
+            for internal mercurial commit function.
+            """
+
+            # check if this path is removed
+            if path in removed:
+                # returning None is a way to mark node for removal
+                return None
+
+            # check if this path is added
+            for node in updated:
+                if node['path'] == path:
+                    return memfilectx(
+                        _repo,
+                        changectx=ctx,
+                        path=node['path'],
+                        data=node['content'],
+                        islink=False,
+                        isexec=bool(node['mode'] & stat.S_IXUSR),
+                        copysource=False)
+
+            raise exceptions.AbortException()(
+                "Given path haven't been marked as added, "
+                "changed or removed (%s)" % path)
+
+        with repo.ui.configoverride({('phases', 'new-commit'): new_commit}):
+
+            commit_ctx = memctx(
+                repo=repo,
+                parents=parents,
+                text=message,
+                files=files,
+                filectxfn=_filectxfn,
+                user=user,
+                date=(commit_time, commit_timezone),
+                extra=extra)
+
+            n = repo.commitctx(commit_ctx)
+            new_id = hex(n)
+
+            return new_id
+
+    @reraise_safe_exceptions
+    def pull(self, wire, url, commit_ids=None):
+        repo = self._factory.repo(wire)
+        # Disable any prompts for this repo
+        repo.ui.setconfig('ui', 'interactive', 'off', '-y')
+
+        remote = peer(repo, {}, url)
+        # Disable any prompts for this remote
+        remote.ui.setconfig('ui', 'interactive', 'off', '-y')
+
+        if commit_ids:
+            commit_ids = [bin(commit_id) for commit_id in commit_ids]
+
+        return exchange.pull(
+            repo, remote, heads=commit_ids, force=None).cgresult
+
+    @reraise_safe_exceptions
+    def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None, hooks=True):
+        repo = self._factory.repo(wire)
+        baseui = self._factory._create_config(wire['config'], hooks=hooks)
+
+        # Mercurial internally has a lot of logic that checks ONLY if
+        # option is defined, we just pass those if they are defined then
+        opts = {}
+        if bookmark:
+            opts['bookmark'] = bookmark
+        if branch:
+            opts['branch'] = branch
+        if revision:
+            opts['rev'] = revision
+
+        commands.pull(baseui, repo, source, **opts)
+
+    @reraise_safe_exceptions
+    def push(self, wire, revisions, dest_path, hooks=True, push_branches=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
         commands.push(baseui, repo, dest=dest_path, rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
+    def strip(self, wire, revision, update, backup):
+        repo = self._factory.repo(wire)
+        ctx = self._get_ctx(repo, revision)
+        hgext_strip(
+            repo.baseui, repo, ctx.node(), update=update, backup=backup)
+
+    @reraise_safe_exceptions
+    def get_unresolved_files(self, wire):
+        repo = self._factory.repo(wire)
+
+        log.debug('Calculating unresolved files for repo: %s', repo)
+        output = io.BytesIO()
+
+        def write(data, **unused_kwargs):
+            output.write(data)
+
+        baseui = self._factory._create_config(wire['config'])
+        baseui.write = write
+
+        commands.resolve(baseui, repo, list=True)
+        unresolved = output.getvalue().splitlines(0)
+        return unresolved
+
+    @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions
     def merge_state(self, wire):
         repo = self._factory.repo(wire)
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
         ms = hg_merge.mergestate(repo)
         return [x for x in ms.unresolved()]
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username, close_branch=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'username', username)
         commands.commit(baseui, repo, message=message, close_branch=close_branch)
 
-
     @reraise_safe_exceptions
     def rebase(self, wire, source=None, dest=None, abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
-        rebase.rebase(
-            baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
+        # In case of sub repositories are used mercurial prompts the user in
+        # case of merge conflicts or different sub repository sources. By
+        # setting the interactive flag to `False` mercurial doesn't prompt the
+        # used but instead uses a default value.
+        repo.ui.setconfig('ui', 'interactive', False)
+        rebase.rebase(baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
+
+    @reraise_safe_exceptions
+    def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone):
+        repo = self._factory.repo(wire)
+        ctx = self._get_ctx(repo, revision)
+        node = ctx.node()
+
+        date = (tag_time, tag_timezone)
+        try:
+            hg_tag.tag(repo, name, node, message, local, user, date)
+        except Abort as e:
+            log.exception("Tag operation aborted")
+            # Exception can contain unicode which we convert
+            raise exceptions.AbortException(e)(repr(e))
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         # we don't need any special hooks for Mercurial
         pass
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         return {
             'pre_version': vcsserver.__version__,
             'post_version': vcsserver.__version__,
         }

vcsserver/hooks.py

@@ -1,711 +1,722 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import logging
 import collections
 import importlib
 import base64
 
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import simplejson as json
 
 from vcsserver import exceptions, subprocessio, settings
-from vcsserver.hgcompat import get_ctx
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         try:
             connection.request('POST', '/', body)
         except Exception:
             log.error('Connection failed on %s', connection)
             raise
         response = connection.getresponse()
 
         response_data = response.read()
 
         try:
             return json.loads(response_data)
         except Exception:
             log.exception('Failed to decode hook response json data. '
                           'response_code:%s, raw_data:%s',
                           response.status, response_data)
             raise
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
+class HooksShadowRepoClient(object):
+
+    def __call__(self, hook_name, extras):
+        return {'output': '', 'status': 0}
+
+
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 class SvnMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to svn clients."""
 
     def __init__(self, stderr=None):
         # SVN needs data sent to stderr for back-to-client messaging
         self.stderr = stderr or sys.stderr
 
     def write(self, message):
         self.stderr.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException()(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
-    if 'hooks_uri' in extras:
-        protocol = extras.get('hooks_protocol')
+    hooks_uri = extras.get('hooks_uri')
+    is_shadow_repo = extras.get('is_shadow_repo')
+    if hooks_uri:
         return HooksHttpClient(extras['hooks_uri'])
+    elif is_shadow_repo:
+        return HooksShadowRepoClient()
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks_client = _get_hooks_client(extras)
     log.debug('Hooks, using client:%s', hooks_client)
     result = hooks_client(hook_name, extras)
     log.debug('Hooks got result: %s', result)
 
     _handle_exception(result)
     writer.write(result['output'])
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node, check_heads=False):
+    from vcsserver.hgcompat import get_ctx
 
     commits = []
     revs = []
     start = get_ctx(repo, node).rev()
     end = len(repo)
     for rev in range(start, end):
         revs.append(rev)
         ctx = get_ctx(repo, rev)
         commit_id = mercurial.node.hex(ctx.node())
         branch = ctx.branch()
         commits.append((commit_id, branch))
 
     parent_heads = []
     if check_heads:
         parent_heads = _check_heads(repo, start, end, revs)
     return commits, parent_heads
 
 
 def _check_heads(repo, start, end, commits):
+    from vcsserver.hgcompat import get_ctx
     changelog = repo.changelog
     parents = set()
 
     for new_rev in commits:
         for p in changelog.parentrevs(new_rev):
             if p == mercurial.node.nullrev:
                 continue
             if p < start:
                 parents.add(p)
 
     for p in parents:
         branch = get_ctx(repo, p).branch()
         # The heads descending from that parent, on the same branch
         parent_heads = set([p])
         reachable = set([p])
         for x in xrange(p + 1, end):
             if get_ctx(repo, x).branch() != branch:
                 continue
             for pp in changelog.parentrevs(x):
                 if pp in reachable:
                     reachable.add(x)
                     parent_heads.discard(pp)
                     parent_heads.add(x)
         # More than one head? Suggest merging
         if len(parent_heads) > 1:
             return list(parent_heads)
 
     return []
 
 
 def _get_git_env():
     env = {}
     for k, v in os.environ.items():
         if k.startswith('GIT'):
             env[k] = v
 
     # serialized version
     return [(k, v) for k, v in env.items()]
 
 
 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
     env = {}
     for k, v in os.environ.items():
         if k.startswith('HG'):
             env[k] = v
 
     env['HG_NODE'] = old_rev
     env['HG_NODE_LAST'] = new_rev
     env['HG_TXNID'] = txnid
     env['HG_PENDING'] = repo_path
 
     return [(k, v) for k, v in env.items()]
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook
     """
     extras = _extras_from_ui(ui)
     detect_force_push = extras.get('detect_force_push')
 
     rev_data = []
     if node and kwargs.get('hooktype') == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
         for commit_id, branch in commits:
             branches[branch].append(commit_id)
 
         for branch, commits in branches.items():
             old_rev = kwargs.get('node_last') or commits[0]
             rev_data.append({
                 'total_commits': len(commits),
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
         for push_ref in rev_data:
             push_ref['multiple_heads'] = _heads
 
             repo_path = os.path.join(
                 extras.get('repo_store', ''), extras.get('repository', ''))
             push_ref['hg_env'] = _get_hg_env(
                 old_rev=push_ref['old_rev'],
                 new_rev=push_ref['new_rev'], txnid=kwargs.get('txnid'),
                 repo_path=repo_path)
 
     extras['hook_type'] = kwargs.get('hooktype', 'pre_push')
     extras['commit_ids'] = rev_data
 
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     """
     Mercurial pre_push hook for SSH
     """
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook
     """
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
 
     commits, _heads = _rev_range_hash(repo, node)
     for commit_id, branch in commits:
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_branches'):
         bookmarks = ui._rc_pushkey_branches
 
     extras['hook_type'] = kwargs.get('hooktype', 'post_push')
     extras['commit_ids'] = commit_ids
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     """
     Mercurial post_push hook for SSH
     """
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
+    from vcsserver.hgcompat import get_ctx
     if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_branches = get_ctx(repo, kwargs['key']).bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 # NOTE(marcink):
                 # we're unable to tell total_commits for git at this point
                 # but we set the variable for consistency with GIT
                 'total_commits': -1,
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     empty_commit_id = '0' * 40
 
     detect_force_push = extras.get('detect_force_push')
 
     for push_ref in rev_data:
         # store our git-env which holds the temp store
         push_ref['git_env'] = _get_git_env()
         push_ref['pruned_sha'] = ''
         if not detect_force_push:
             # don't check for forced-push when we don't need to
             continue
 
         type_ = push_ref['type']
         new_branch = push_ref['old_rev'] == empty_commit_id
         delete_branch = push_ref['new_rev'] == empty_commit_id
         if type_ == 'heads' and not (new_branch or delete_branch):
             old_rev = push_ref['old_rev']
             new_rev = push_ref['new_rev']
             cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, '^{}'.format(new_rev)]
             stdout, stderr = subprocessio.run_command(
                 cmd, env=os.environ.copy())
             # means we're having some non-reachable objects, this forced push was used
             if stdout:
                 push_ref['pruned_sha'] = stdout.splitlines()
 
     extras['hook_type'] = 'pre_receive'
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
                 # starting new branch case
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 # Fix up head revision if needed
                 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
                 try:
                     subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
                     cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
                     print("Setting default branch to %s" % push_ref['name'])
                     subprocessio.run_command(cmd, env=os.environ.copy())
 
                 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
                        '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 heads = stdout
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head
                                  in heads.splitlines() if head) or '.'
                 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
                        '--pretty=format:%H', '--', push_ref['new_rev'],
                        '--not', heads]
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE, 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = subprocessio.run_command(
                     cmd, env=os.environ.copy())
                 git_revs.extend(stdout.splitlines())
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['hook_type'] = 'post_receive'
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())
 
 
 def _get_extras_from_txn_id(path, txn_id):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-t', txn_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from txn_id')
 
     return extras
 
 
 def _get_extras_from_commit_id(commit_id, path):
     extras = {}
     try:
         cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
                '-r', commit_id,
                '--revprop', path, 'rc-scm-extras']
         stdout, stderr = subprocessio.run_command(
             cmd, env=os.environ.copy())
         extras = json.loads(base64.urlsafe_b64decode(stdout))
     except Exception:
         log.exception('Failed to extract extras info from commit_id')
 
     return extras
 
 
 def svn_pre_commit(repo_path, commit_data, env):
     path, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_txn_id(path, txn_id)
         if not extras:
             return 0
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = [txn_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'total_commits': 1,
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     return _call_hook('pre_push', extras, SvnMessageWriter())
 
 
 def svn_post_commit(repo_path, commit_data, env):
     """
     commit_data is path, rev, txn_id
     """
     path, commit_id, txn_id = commit_data
     branches = []
     tags = []
 
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
         # fallback method to read from TXN-ID stored data
         extras = _get_extras_from_commit_id(commit_id, path)
         if not extras:
             return 0
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]
     extras['txn_id'] = txn_id
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
         'total_commits': 1,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, SvnMessageWriter())
         except Exception:
             pass
 
     return _call_hook('post_push', extras, SvnMessageWriter())

vcsserver/http_main.py

@@ -1,610 +1,675 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import sys
 import base64
 import locale
 import logging
 import uuid
 import wsgiref.util
 import traceback
 import tempfile
 from itertools import chain
+from cStringIO import StringIO
 
 import simplejson as json
 import msgpack
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.compat import configparser
+from pyramid.response import Response
 
+from vcsserver.utils import safe_int
 
 log = logging.getLogger(__name__)
 
 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
 
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
     log.error(
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
 
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
 
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 
 def _is_request_chunked(environ):
     stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
     return stream
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
     return settings[name]
 
 
 def _bool_setting(settings, name, default):
     input_val = settings.get(name, default)
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
     return settings[name]
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     # Otherwise we assume it uses pyramids space/newline separation.
     settings[name] = aslist(raw_value)
     return settings[name]
 
 
 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
 
     if default_when_empty and not value:
         # use default value when value is empty
         value = default
 
     if lower:
         value = value.lower()
     settings[name] = value
     return settings[name]
 
 
 class VCS(object):
-    def __init__(self, locale=None, cache_config=None):
-        self.locale = locale
+    def __init__(self, locale_conf=None, cache_config=None):
+        self.locale = locale_conf
         self.cache_config = cache_config
         self._configure_locale()
 
         if GitFactory and GitRemote:
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
 
             # hg factory is used for svn url validation
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.info("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 def not_found(request):
     return {'status': '404 NOT FOUND'}
 
 
 class VCSViewPredicate(object):
     def __init__(self, val, config):
         self.remotes = val
 
     def text(self):
         return 'vcs view method = %s' % (self.remotes.keys(),)
 
     phash = text
 
     def __call__(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self.remotes
 
 
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
         self._sanitize_settings_and_apply_defaults(settings)
 
         self.config = Configurator(settings=settings)
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
 
-        locale = settings.get('locale', '') or 'en_US.UTF-8'
-        vcs = VCS(locale=locale, cache_config=settings)
+        settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
+        vcs = VCS(locale_conf=settings_locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
         self._configure_settings(global_config, settings)
         self._configure()
 
     def _configure_settings(self, global_config, app_settings):
         """
         Configure the settings module.
         """
         settings_merged = global_config.copy()
         settings_merged.update(app_settings)
 
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
         binary_dir = app_settings.get('core.binary_dir', None)
         if binary_dir:
             settings.BINARY_DIR = binary_dir
 
         # Store the settings to make them available to other modules.
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
     def _sanitize_settings_and_apply_defaults(self, settings):
         temp_store = tempfile.gettempdir()
         default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
         # save default, cache dir, and use it for all backends later.
         default_cache_dir = _string_setting(
             settings,
             'cache_dir',
             default_cache_dir, lower=False, default_when_empty=True)
 
         # ensure we have our dir created
         if not os.path.isdir(default_cache_dir):
             os.makedirs(default_cache_dir, mode=0o755)
 
         # exception store cache
         _string_setting(
             settings,
             'exception_tracker.store_path',
             temp_store, lower=False, default_when_empty=True)
 
         # repo_object cache
         _string_setting(
             settings,
             'rc_cache.repo_object.backend',
-            'dogpile.cache.rc.memory_lru')
+            'dogpile.cache.rc.file_namespace', lower=False)
         _int_setting(
             settings,
             'rc_cache.repo_object.expiration_time',
-            300)
-        _int_setting(
+            30 * 24 * 60 * 60)
+        _string_setting(
             settings,
-            'rc_cache.repo_object.max_size',
-            1024)
+            'rc_cache.repo_object.arguments.filename',
+            os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
 
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
+
+        # rpc methods
         self.config.add_route('vcs', '/{backend}')
+
+        # streaming rpc remote methods
+        self.config.add_route('vcs_stream', '/{backend}/stream')
+
+        # vcs operations clone/push as streaming
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(self.status_view, route_name='status', renderer='json')
         self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
                              vcs_view=self._remotes)
+        self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
+                             vcs_view=self._remotes)
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         self.config.add_view_predicate('vcs_view', VCSViewPredicate)
 
         self.config.add_notfound_view(not_found, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
-            'vcsserver.tweens.RequestWrapperTween',
+            'vcsserver.tweens.request_wrapper.RequestWrapperTween',
         )
+        self.config.add_request_method(
+            'vcsserver.lib.request_counter.get_request_counter',
+            'request_count')
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
-    def vcs_view(self, request):
+    def _vcs_view_params(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
-        params = payload.get('params')
+        params = payload['params']
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         context_uid = None
 
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
+        repo_state_uid = wire.get('repo_state_uid') if wire else None
 
-        log.debug('method called:%s with kwargs:%s context_uid: %s',
-                  method, kwargs, context_uid)
+        # NOTE(marcink): trading complexity for slight performance
+        if log.isEnabledFor(logging.DEBUG):
+            no_args_methods = [
+                'archive_repo'
+            ]
+            if method in no_args_methods:
+                call_args = ''
+            else:
+                call_args = args[1:]
+
+            log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
+                      method, call_args, kwargs, context_uid, repo_state_uid)
+
+        return payload, remote, method, args, kwargs
+
+    def vcs_view(self, request):
+
+        payload, remote, method, args, kwargs = self._vcs_view_params(request)
+        payload_id = payload.get('id')
+
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             exc_info = list(sys.exc_info())
             exc_type, exc_value, exc_traceback = exc_info
 
             org_exc = getattr(e, '_org_exc', None)
             org_exc_name = None
             org_exc_tb = ''
             if org_exc:
                 org_exc_name = org_exc.__class__.__name__
                 org_exc_tb = getattr(e, '_org_exc_tb', '')
                 # replace our "faked" exception with our org
                 exc_info[0] = org_exc.__class__
                 exc_info[1] = org_exc
 
             store_exception(id(exc_info), exc_info)
 
             tb_info = ''.join(
                 traceback.format_exception(exc_type, exc_value, exc_traceback))
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
-                'id': payload.get('id'),
+                'id': payload_id,
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'org_exc_tb': org_exc_tb,
                     'type': type_
                 }
             }
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:
                 pass
         else:
             resp = {
-                'id': payload.get('id'),
+                'id': payload_id,
                 'result': resp
             }
 
         return resp
 
+    def vcs_stream_view(self, request):
+        payload, remote, method, args, kwargs = self._vcs_view_params(request)
+        # this method has a stream: marker we remove it here
+        method = method.split('stream:')[-1]
+        chunk_size = safe_int(payload.get('chunk_size')) or 4096
+
+        try:
+            resp = getattr(remote, method)(*args, **kwargs)
+        except Exception as e:
+            raise
+
+        def get_chunked_data(method_resp):
+            stream = StringIO(method_resp)
+            while 1:
+                chunk = stream.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+
+        response = Response(app_iter=get_chunked_data(resp))
+        response.content_type = 'application/octet-stream'
+
+        return response
+
     def status_view(self, request):
         import vcsserver
         return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
                 'pid': os.getpid()}
 
     def service_view(self, request):
         import vcsserver
 
         payload = msgpack.unpackb(request.body, use_list=True)
+        server_config, app_config = {}, {}
 
         try:
             path = self.global_config['__file__']
-            config = configparser.ConfigParser()
+            config = configparser.RawConfigParser()
+
             config.read(path)
-            parsed_ini = config
-            if parsed_ini.has_section('server:main'):
-                parsed_ini = dict(parsed_ini.items('server:main'))
+
+            if config.has_section('server:main'):
+                server_config = dict(config.items('server:main'))
+            if config.has_section('app:main'):
+                app_config = dict(config.items('app:main'))
+
         except Exception:
             log.exception('Failed to read .ini file for display')
-            parsed_ini = {}
+
+        environ = os.environ.items()
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
-                config=parsed_ini,
+                config=server_config,
+                app_config=app_config,
+                environ=environ,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
-            value = msgpack.packb(value)
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
-            return value
+            return msgpack.packb(value)
         return _render
 
     def set_env_from_config(self, environ, config):
         dict_conf = {}
         try:
             for elem in config:
                 if elem[0] == 'rhodecode':
                     dict_conf = json.loads(elem[2])
                     break
         except Exception:
             log.exception('Failed to fetch SCM CONFIG')
             return
 
         username = dict_conf.get('username')
         if username:
             environ['REMOTE_USER'] = username
             # mercurial specific, some extension api rely on this
             environ['HGUSER'] = username
 
         ip = dict_conf.get('ip')
         if ip:
             environ['REMOTE_HOST'] = ip
 
         if _is_request_chunked(environ):
             # set the compatibility flag for webob
             environ['wsgi.input_terminated'] = True
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
 
                 path = environ['PATH_INFO']
                 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
                 log.debug(
                     'LFS: Detecting if request `%s` is LFS server path based '
                     'on content type:`%s`, is_lfs:%s',
                     path, content_type, is_lfs_request)
 
                 if not is_lfs_request:
                     # fallback detection by path
                     if GIT_LFS_PROTO_PAT.match(path):
                         is_lfs_request = True
                     log.debug(
                         'LFS: fallback detection by path of: `%s`, is_lfs:%s',
                         path, is_lfs_request)
 
                 if is_lfs_request:
                     app = scm_app.create_git_lfs_wsgi_app(
                         repo_path, repo_name, config)
                 else:
                     app = scm_app.create_git_wsgi_app(
                         repo_path, repo_name, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
 
                 return app(environ, start_response)
 
             return _git_stream
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
         if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
             return HTTPRepoBranchProtected(title=exception.message)
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
 
         traceback_info = 'unavailable'
         if request.exc_info:
             exc_type, exc_value, exc_tb = request.exc_info
             traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
 
         log.error(
             'error occurred handling this request for path: %s, \n tb: %s',
             request.path, traceback_info)
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
     app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/lib/rc_cache/__init__.py

@@ -1,60 +1,72 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 from dogpile.cache import register_backend
 
 register_backend(
     "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
+register_backend(
+    "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
+    "FileNamespaceBackend")
+
+register_backend(
+    "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
+    "RedisPickleBackend")
+
+register_backend(
+    "dogpile.cache.rc.redis_msgpack", "vcsserver.lib.rc_cache.backends",
+    "RedisMsgPackBackend")
+
+
 log = logging.getLogger(__name__)
 
 from . import region_meta
-from .util import key_generator, get_default_cache_settings, make_region
+from .utils import (get_default_cache_settings, backend_key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         avail_regions.add(namespace_name)
     log.debug('dogpile: found following cache regions: %s', avail_regions)
 
     # register them into namespace
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
-            function_key_generator=key_generator
+            function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
-
-        log.debug('dogpile: registering a new region %s[%s]',
-                  region_name, new_region.__dict__)
+        new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
+        log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

vcsserver/lib/rc_cache/backends.py

@@ -1,51 +1,253 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import time
+import errno
 import logging
 
+import msgpack
+import redis
+
+from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
+from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import redis as redis_backend
+from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
+from dogpile.cache.util import memoized_property
+
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
+    key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
+
+
+class PickleSerializer(object):
+
+    def _dumps(self, value, safe=False):
+        try:
+            return compat.pickle.dumps(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        try:
+            return compat.pickle.loads(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
+class MsgPackSerializer(object):
+
+    def _dumps(self, value, safe=False):
+        try:
+            return msgpack.packb(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        """
+        pickle maintained the `CachedValue` wrapper of the tuple
+        msgpack does not, so it must be added back in.
+       """
+        try:
+            value = msgpack.unpackb(value, use_list=False)
+            return CachedValue(*value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
+import fcntl
+flock_org = fcntl.flock
+
+
+class CustomLockFactory(FileLock):
+
+    pass
+
+
+class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
+    key_prefix = 'file_backend'
+
+    def __init__(self, arguments):
+        arguments['lock_factory'] = CustomLockFactory
+        super(FileNamespaceBackend, self).__init__(arguments)
+
+    def __repr__(self):
+        return '{} `{}`'.format(self.__class__, self.filename)
+
+    def list_keys(self, prefix=''):
+        prefix = '{}:{}'.format(self.key_prefix, prefix)
+
+        def cond(v):
+            if not prefix:
+                return True
+
+            if v.startswith(prefix):
+                return True
+            return False
+
+        with self._dbm_file(True) as dbm:
+
+            return filter(cond, dbm.keys())
+
+    def get_store(self):
+        return self.filename
+
+    def get(self, key):
+        with self._dbm_file(False) as dbm:
+            if hasattr(dbm, 'get'):
+                value = dbm.get(key, NO_VALUE)
+            else:
+                # gdbm objects lack a .get method
+                try:
+                    value = dbm[key]
+                except KeyError:
+                    value = NO_VALUE
+            if value is not NO_VALUE:
+                value = self._loads(value)
+            return value
+
+    def set(self, key, value):
+        with self._dbm_file(True) as dbm:
+            dbm[key] = self._dumps(value)
+
+    def set_multi(self, mapping):
+        with self._dbm_file(True) as dbm:
+            for key, value in mapping.items():
+                dbm[key] = self._dumps(value)
+
+
+class BaseRedisBackend(redis_backend.RedisBackend):
+
+    def _create_client(self):
+        args = {}
+
+        if self.url is not None:
+            args.update(url=self.url)
+
+        else:
+            args.update(
+                host=self.host, password=self.password,
+                port=self.port, db=self.db
+            )
+
+        connection_pool = redis.ConnectionPool(**args)
+
+        return redis.StrictRedis(connection_pool=connection_pool)
+
+    def list_keys(self, prefix=''):
+        prefix = '{}:{}*'.format(self.key_prefix, prefix)
+        return self.client.keys(prefix)
+
+    def get_store(self):
+        return self.client.connection_pool
+
+    def get(self, key):
+        value = self.client.get(key)
+        if value is None:
+            return NO_VALUE
+        return self._loads(value)
+
+    def get_multi(self, keys):
+        if not keys:
+            return []
+        values = self.client.mget(keys)
+        loads = self._loads
+        return [
+            loads(v) if v is not None else NO_VALUE
+            for v in values]
+
+    def set(self, key, value):
+        if self.redis_expiration_time:
+            self.client.setex(key, self.redis_expiration_time,
+                              self._dumps(value))
+        else:
+            self.client.set(key, self._dumps(value))
+
+    def set_multi(self, mapping):
+        dumps = self._dumps
+        mapping = dict(
+            (k, dumps(v))
+            for k, v in mapping.items()
+        )
+
+        if not self.redis_expiration_time:
+            self.client.mset(mapping)
+        else:
+            pipe = self.client.pipeline()
+            for key, value in mapping.items():
+                pipe.setex(key, self.redis_expiration_time, value)
+            pipe.execute()
+
+    def get_mutex(self, key):
+        u = redis_backend.u
+        if self.distributed_lock:
+            lock_key = u('_lock_{0}').format(key)
+            log.debug('Trying to acquire Redis lock for key %s', lock_key)
+            return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
+        else:
+            return None
+
+
+class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
+    key_prefix = 'redis_pickle_backend'
+    pass
+
+
+class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
+    key_prefix = 'redis_msgpack_backend'
+    pass

vcsserver/lib/rc_cache/utils.py

@@ -1,136 +1,153 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import logging
 import functools
+from decorator import decorate
+
+from dogpile.cache import CacheRegion
+from dogpile.cache.util import compat
 
 from vcsserver.utils import safe_str, sha1
-from dogpile.cache import CacheRegion
-from dogpile.cache.util import compat
+
 
 log = logging.getLogger(__name__)
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=compat.string_type,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different than should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = compat.callable(expiration_time)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
-        def decorator(fn):
+        def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
+
+            if not condition:
+                log.debug('Calling un-cached func:%s', user_func.func_name)
+                return user_func(*arg, **kw)
+
+            key = key_generator(*arg, **kw)
+
+            timeout = expiration_time() if expiration_time_is_callable \
+                else expiration_time
+
+            log.debug('Calling cached fn:%s', user_func.func_name)
+            return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
+
+        def cache_decorator(user_func):
             if to_str is compat.string_type:
                 # backwards compatible
-                key_generator = function_key_generator(namespace, fn)
+                key_generator = function_key_generator(namespace, user_func)
             else:
-                key_generator = function_key_generator(namespace, fn, to_str=to_str)
-
-            @functools.wraps(fn)
-            def decorate(*arg, **kw):
-                key = key_generator(*arg, **kw)
+                key_generator = function_key_generator(namespace, user_func, to_str=to_str)
 
-                @functools.wraps(fn)
-                def creator():
-                    return fn(*arg, **kw)
-
-                if not condition:
-                    return creator()
-
-                timeout = expiration_time() if expiration_time_is_callable \
-                    else expiration_time
-
-                return self.get_or_create(key, creator, timeout, should_cache_fn)
+            def refresh(*arg, **kw):
+                """
+                Like invalidate, but regenerates the value instead
+                """
+                key = key_generator(*arg, **kw)
+                value = user_func(*arg, **kw)
+                self.set(key, value)
+                return value
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
-            def refresh(*arg, **kw):
-                key = key_generator(*arg, **kw)
-                value = fn(*arg, **kw)
-                self.set(key, value)
-                return value
+            user_func.set = set_
+            user_func.invalidate = invalidate
+            user_func.get = get
+            user_func.refresh = refresh
+            user_func.key_generator = key_generator
+            user_func.original = user_func
 
-            decorate.set = set_
-            decorate.invalidate = invalidate
-            decorate.refresh = refresh
-            decorate.get = get
-            decorate.original = fn
-            decorate.key_generator = key_generator
+            # Use `decorate` to preserve the signature of :param:`user_func`.
 
-            return decorate
+            return decorate(user_func, functools.partial(
+                get_or_create_for_user_func, key_generator))
 
-        return decorator
+        return cache_decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
-                if isinstance(val, basestring):
+                if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
-def key_generator(namespace, fn):
+def backend_key_generator(backend):
+    """
+    Special wrapper that also sends over the backend to the key generator
+    """
+    def wrapper(namespace, fn):
+        return key_generator(backend, namespace, fn)
+    return wrapper
+
+
+def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
-        namespace_pref = namespace or 'default'
+        backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
+        namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
-        final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+        final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key

vcsserver/subprocessio.py

@@ -1,523 +1,519 b''
 """
 Module provides a class allowing to wrap communication over subprocess.Popen
 input, output, error streams into a meaningfull, non-blocking, concurrent
 stream processor exposing the output data as an iterator fitting to be a
 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
 
 Copyright (c) 2011  Daniel Dotsenko <dotsa[at]hotmail.com>
 
 This file is part of git_http_backend.py Project.
 
 git_http_backend.py Project is free software: you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public License as
 published by the Free Software Foundation, either version 2.1 of the License,
 or (at your option) any later version.
 
 git_http_backend.py Project is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 GNU Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public License
 along with git_http_backend.py Project.
 If not, see <http://www.gnu.org/licenses/>.
 """
 import os
 import logging
 import subprocess32 as subprocess
 from collections import deque
 from threading import Event, Thread
 
 log = logging.getLogger(__name__)
 
 
 class StreamFeeder(Thread):
     """
     Normal writing into pipe-like is blocking once the buffer is filled.
     This thread allows a thread to seep data from a file-like into a pipe
     without blocking the main thread.
     We close inpipe once the end of the source stream is reached.
     """
 
     def __init__(self, source):
         super(StreamFeeder, self).__init__()
         self.daemon = True
         filelike = False
         self.bytes = bytes()
         if type(source) in (type(''), bytes, bytearray):  # string-like
             self.bytes = bytes(source)
         else:  # can be either file pointer or file-like
             if type(source) in (int, long):  # file pointer it is
                 # converting file descriptor (int) stdin into file-like
                 try:
                     source = os.fdopen(source, 'rb', 16384)
                 except Exception:
                     pass
             # let's see if source is file-like by now
             try:
                 filelike = source.read
             except Exception:
                 pass
         if not filelike and not self.bytes:
             raise TypeError("StreamFeeder's source object must be a readable "
                             "file-like, a file descriptor, or a string-like.")
         self.source = source
         self.readiface, self.writeiface = os.pipe()
 
     def run(self):
         t = self.writeiface
         try:
             if self.bytes:
                 os.write(t, self.bytes)
             else:
                 s = self.source
                 b = s.read(4096)
                 while b:
                     os.write(t, b)
                     b = s.read(4096)
         finally:
             os.close(t)
 
     @property
     def output(self):
         return self.readiface
 
 
 class InputStreamChunker(Thread):
     def __init__(self, source, target, buffer_size, chunk_size):
 
         super(InputStreamChunker, self).__init__()
 
         self.daemon = True  # die die die.
 
         self.source = source
         self.target = target
         self.chunk_count_max = int(buffer_size / chunk_size) + 1
         self.chunk_size = chunk_size
 
         self.data_added = Event()
         self.data_added.clear()
 
         self.keep_reading = Event()
         self.keep_reading.set()
 
         self.EOF = Event()
         self.EOF.clear()
 
         self.go = Event()
         self.go.set()
 
     def stop(self):
         self.go.clear()
         self.EOF.set()
         try:
             # this is not proper, but is done to force the reader thread let
             # go of the input because, if successful, .close() will send EOF
             # down the pipe.
             self.source.close()
         except:
             pass
 
     def run(self):
         s = self.source
         t = self.target
         cs = self.chunk_size
         chunk_count_max = self.chunk_count_max
         keep_reading = self.keep_reading
         da = self.data_added
         go = self.go
 
         try:
             b = s.read(cs)
         except ValueError:
             b = ''
 
         timeout_input = 20
         while b and go.is_set():
             if len(t) > chunk_count_max:
                 keep_reading.clear()
                 keep_reading.wait(timeout_input)
                 if len(t) > chunk_count_max + timeout_input:
                     log.error("Timed out while waiting for input from subprocess.")
                     os._exit(-1)  # this will cause the worker to recycle itself
 
             t.append(b)
             da.set()
 
             try:
                 b = s.read(cs)
             except ValueError:
                 b = ''
 
         self.EOF.set()
         da.set()  # for cases when done but there was no input.
 
 
 class BufferedGenerator(object):
     """
     Class behaves as a non-blocking, buffered pipe reader.
     Reads chunks of data (through a thread)
     from a blocking pipe, and attaches these to an array (Deque) of chunks.
     Reading is halted in the thread when max chunks is internally buffered.
     The .next() may operate in blocking or non-blocking fashion by yielding
     '' if no data is ready
     to be sent or by not returning until there is some data to send
     When we get EOF from underlying source pipe we raise the marker to raise
     StopIteration after the last chunk of data is yielded.
     """
 
     def __init__(self, source, buffer_size=65536, chunk_size=4096,
                  starting_values=None, bottomless=False):
         starting_values = starting_values or []
 
         if bottomless:
             maxlen = int(buffer_size / chunk_size)
         else:
             maxlen = None
 
         self.data = deque(starting_values, maxlen)
         self.worker = InputStreamChunker(source, self.data, buffer_size,
                                          chunk_size)
         if starting_values:
             self.worker.data_added.set()
         self.worker.start()
 
     ####################
     # Generator's methods
     ####################
 
     def __iter__(self):
         return self
 
     def next(self):
         while not len(self.data) and not self.worker.EOF.is_set():
             self.worker.data_added.clear()
             self.worker.data_added.wait(0.2)
         if len(self.data):
             self.worker.keep_reading.set()
             return bytes(self.data.popleft())
         elif self.worker.EOF.is_set():
             raise StopIteration
 
     def throw(self, exc_type, value=None, traceback=None):
         if not self.worker.EOF.is_set():
             raise exc_type(value)
 
     def start(self):
         self.worker.start()
 
     def stop(self):
         self.worker.stop()
 
     def close(self):
         try:
             self.worker.stop()
             self.throw(GeneratorExit)
         except (GeneratorExit, StopIteration):
             pass
 
-    def __del__(self):
-        self.close()
-
     ####################
     # Threaded reader's infrastructure.
     ####################
     @property
     def input(self):
         return self.worker.w
 
     @property
     def data_added_event(self):
         return self.worker.data_added
 
     @property
     def data_added(self):
         return self.worker.data_added.is_set()
 
     @property
     def reading_paused(self):
         return not self.worker.keep_reading.is_set()
 
     @property
     def done_reading_event(self):
         """
         Done_reding does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
 
         :returns: An Event class instance.
         """
         return self.worker.EOF
 
     @property
     def done_reading(self):
         """
         Done_reding does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
 
         :returns: An Bool value.
         """
         return self.worker.EOF.is_set()
 
     @property
     def length(self):
         """
         returns int.
 
         This is the lenght of the que of chunks, not the length of
         the combined contents in those chunks.
 
         __len__() cannot be meaningfully implemented because this
         reader is just flying throuh a bottomless pit content and
         can only know the lenght of what it already saw.
 
         If __len__() on WSGI server per PEP 3333 returns a value,
         the responce's length will be set to that. In order not to
         confuse WSGI PEP3333 servers, we will not implement __len__
         at all.
         """
         return len(self.data)
 
     def prepend(self, x):
         self.data.appendleft(x)
 
     def append(self, x):
         self.data.append(x)
 
     def extend(self, o):
         self.data.extend(o)
 
     def __getitem__(self, i):
         return self.data[i]
 
 
 class SubprocessIOChunker(object):
     """
     Processor class wrapping handling of subprocess IO.
 
     .. important::
 
        Watch out for the method `__del__` on this class. If this object
        is deleted, it will kill the subprocess, so avoid to
        return the `output` attribute or usage of it like in the following
        example::
 
           # `args` expected to run a program that produces a lot of output
           output = ''.join(SubprocessIOChunker(
              args, shell=False, inputstream=inputstream, env=environ).output)
 
           # `output` will not contain all the data, because the __del__ method
           # has already killed the subprocess in this case before all output
           # has been consumed.
 
 
 
     In a way, this is a "communicate()" replacement with a twist.
 
     - We are multithreaded. Writing in and reading out, err are all sep threads.
     - We support concurrent (in and out) stream processing.
     - The output is not a stream. It's a queue of read string (bytes, not unicode)
       chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
     - We are non-blocking in more respects than communicate()
       (reading from subprocess out pauses when internal buffer is full, but
        does not block the parent calling code. On the flip side, reading from
        slow-yielding subprocess may block the iteration until data shows up. This
        does not block the parallel inpipe reading occurring parallel thread.)
 
     The purpose of the object is to allow us to wrap subprocess interactions into
     and interable that can be passed to a WSGI server as the application's return
     value. Because of stream-processing-ability, WSGI does not have to read ALL
     of the subprocess's output and buffer it, before handing it to WSGI server for
     HTTP response. Instead, the class initializer reads just a bit of the stream
     to figure out if error ocurred or likely to occur and if not, just hands the
     further iteration over subprocess output to the server for completion of HTTP
     response.
 
     The real or perceived subprocess error is trapped and raised as one of
     EnvironmentError family of exceptions
 
     Example usage:
     #    try:
     #        answer = SubprocessIOChunker(
     #            cmd,
     #            input,
     #            buffer_size = 65536,
     #            chunk_size = 4096
     #            )
     #    except (EnvironmentError) as e:
     #        print str(e)
     #        raise e
     #
     #    return answer
 
 
     """
 
     # TODO: johbo: This is used to make sure that the open end of the PIPE
     # is closed in the end. It would be way better to wrap this into an
     # object, so that it is closed automatically once it is consumed or
     # something similar.
     _close_input_fd = None
 
     _closed = False
 
     def __init__(self, cmd, inputstream=None, buffer_size=65536,
                  chunk_size=4096, starting_values=None, fail_on_stderr=True,
                  fail_on_return_code=True, **kwargs):
         """
         Initializes SubprocessIOChunker
 
         :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
         :param inputstream: (Default: None) A file-like, string, or file pointer.
         :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
         :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
         :param starting_values: (Default: []) An array of strings to put in front of output que.
         :param fail_on_stderr: (Default: True) Whether to raise an exception in
                                case something is written to stderr.
         :param fail_on_return_code: (Default: True) Whether to raise an
                                     exception if the return code is not 0.
         """
 
         starting_values = starting_values or []
         if inputstream:
             input_streamer = StreamFeeder(inputstream)
             input_streamer.start()
             inputstream = input_streamer.output
             self._close_input_fd = inputstream
 
         self._fail_on_stderr = fail_on_stderr
         self._fail_on_return_code = fail_on_return_code
 
         _shell = kwargs.get('shell', True)
         kwargs['shell'] = _shell
 
         _p = subprocess.Popen(cmd, bufsize=-1,
                               stdin=inputstream,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               **kwargs)
 
         bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size,
                                    starting_values)
         bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)
 
         while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
             # doing this until we reach either end of file, or end of buffer.
             bg_out.data_added_event.wait(1)
             bg_out.data_added_event.clear()
 
         # at this point it's still ambiguous if we are done reading or just full buffer.
         # Either way, if error (returned by ended process, or implied based on
         # presence of stuff in stderr output) we error out.
         # Else, we are happy.
         _returncode = _p.poll()
 
         if ((_returncode and fail_on_return_code) or
                 (fail_on_stderr and _returncode is None and bg_err.length)):
             try:
                 _p.terminate()
             except Exception:
                 pass
             bg_out.stop()
             bg_err.stop()
             if fail_on_stderr:
                 err = ''.join(bg_err)
                 raise EnvironmentError(
                     "Subprocess exited due to an error:\n" + err)
             if _returncode and fail_on_return_code:
                 err = ''.join(bg_err)
                 if not err:
                     # maybe get empty stderr, try stdout instead
                     # in many cases git reports the errors on stdout too
                     err = ''.join(bg_out)
                 raise EnvironmentError(
                     "Subprocess exited with non 0 ret code:%s: stderr:%s" % (
                         _returncode, err))
 
         self.process = _p
         self.output = bg_out
         self.error = bg_err
         self.inputstream = inputstream
 
     def __iter__(self):
         return self
 
     def next(self):
         # Note: mikhail: We need to be sure that we are checking the return
         # code after the stdout stream is closed. Some processes, e.g. git
         # are doing some magic in between closing stdout and terminating the
         # process and, as a result,  we are not getting return code on "slow"
         # systems.
         result = None
         stop_iteration = None
         try:
             result = self.output.next()
         except StopIteration as e:
             stop_iteration = e
 
         if self.process.poll() and self._fail_on_return_code:
             err = '%s' % ''.join(self.error)
             raise EnvironmentError(
                 "Subprocess exited due to an error:\n" + err)
 
         if stop_iteration:
             raise stop_iteration
         return result
 
     def throw(self, type, value=None, traceback=None):
         if self.output.length or not self.output.done_reading:
             raise type(value)
 
     def close(self):
         if self._closed:
             return
         self._closed = True
         try:
             self.process.terminate()
-        except:
+        except Exception:
             pass
         if self._close_input_fd:
             os.close(self._close_input_fd)
         try:
             self.output.close()
-        except:
+        except Exception:
             pass
         try:
             self.error.close()
-        except:
+        except Exception:
             pass
         try:
             os.close(self.inputstream)
-        except:
+        except Exception:
             pass
 
-    def __del__(self):
-        self.close()
-
 
 def run_command(arguments, env=None):
     """
     Run the specified command and return the stdout.
 
     :param arguments: sequence of program arguments (including the program name)
     :type arguments: list[str]
     """
 
     cmd = arguments
     log.debug('Running subprocessio command %s', cmd)
+    proc = None
     try:
         _opts = {'shell': False, 'fail_on_stderr': False}
         if env:
             _opts.update({'env': env})
-        p = SubprocessIOChunker(cmd, **_opts)
-        stdout = ''.join(p)
-        stderr = ''.join(''.join(p.error))
+        proc = SubprocessIOChunker(cmd, **_opts)
+        return ''.join(proc), ''.join(proc.error)
     except (EnvironmentError, OSError) as err:
         cmd = ' '.join(cmd)  # human friendly CMD
         tb_err = ("Couldn't run subprocessio command (%s).\n"
                   "Original error was:%s\n" % (cmd, err))
         log.exception(tb_err)
         raise Exception(tb_err)
+    finally:
+        if proc:
+            proc.close()
 
-    return stdout, stderr

vcsserver/svn.py

@@ -1,775 +1,799 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 from __future__ import absolute_import
 
 import os
 import subprocess
 from urllib2 import URLError
 import urlparse
 import logging
 import posixpath as vcspath
 import StringIO
 import urllib
 import traceback
 
 import svn.client
 import svn.core
 import svn.delta
 import svn.diff
 import svn.fs
 import svn.repos
 
 from vcsserver import svn_diff, exceptions, subprocessio, settings
 from vcsserver.base import RepoFactory, raise_from_original
+from vcsserver.vcs_base import RemoteBase
 
 log = logging.getLogger(__name__)
 
 
 # Set of svn compatible version flags.
 # Compare with subversion/svnadmin/svnadmin.c
 svn_compatible_versions = {
     'pre-1.4-compatible',
     'pre-1.5-compatible',
     'pre-1.6-compatible',
     'pre-1.8-compatible',
     'pre-1.9-compatible'
 }
 
 svn_compatible_versions_map = {
     'pre-1.4-compatible': '1.3',
     'pre-1.5-compatible': '1.4',
     'pre-1.6-compatible': '1.5',
     'pre-1.8-compatible': '1.7',
     'pre-1.9-compatible': '1.8',
 }
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting svn exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in svn remote call")
                 raise_from_original(exceptions.UnhandledException(e))
             raise
     return wrapper
 
 
 class SubversionFactory(RepoFactory):
     repo_type = 'svn'
 
     def _create_repo(self, wire, create, compatible_version):
         path = svn.core.svn_path_canonicalize(wire['path'])
         if create:
             fs_config = {'compatible-version': '1.9'}
             if compatible_version:
                 if compatible_version not in svn_compatible_versions:
                     raise Exception('Unknown SVN compatible version "{}"'
                                     .format(compatible_version))
                 fs_config['compatible-version'] = \
                     svn_compatible_versions_map[compatible_version]
 
             log.debug('Create SVN repo with config "%s"', fs_config)
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
 
         log.debug('Got SVN object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
         """
         Get a repository instance for the given path.
-
-        Uses internally the low level beaker API since the decorators introduce
-        significant overhead.
         """
-        region = self._cache_region
-        context = wire.get('context', None)
-        repo_path = wire.get('path', '')
-        context_uid = '{}'.format(context)
-        cache = wire.get('cache', True)
-        cache_on = context and cache
-
-        @region.conditional_cache_on_arguments(condition=cache_on)
-        def create_new_repo(_repo_type, _repo_path, _context_uid, compatible_version_id):
-            return self._create_repo(wire, create, compatible_version)
-
-        return create_new_repo(self.repo_type, repo_path, context_uid,
-                               compatible_version)
+        return self._create_repo(wire, create, compatible_version)
 
 
 NODE_TYPE_MAPPING = {
     svn.core.svn_node_file: 'file',
     svn.core.svn_node_dir: 'dir',
 }
 
 
-class SvnRemote(object):
+class SvnRemote(RemoteBase):
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
         # TODO: Remove once we do not use internal Mercurial objects anymore
         # for subversion
         self._hg_factory = hg_factory
 
     @reraise_safe_exceptions
     def discover_svn_version(self):
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
         return svn_ver
 
     @reraise_safe_exceptions
     def is_empty(self, wire):
-        repo = self._factory.repo(wire)
 
         try:
             return self.lookup(wire, -1) == 0
         except Exception:
             log.exception("failed to read object_store")
             return False
 
     def check_url(self, url, config_items):
         # this can throw exception if not installed, but we detect this
         from hgsubversion import svnrepo
 
         baseui = self._hg_factory._create_config(config_items)
         # uuid function get's only valid UUID from proper repo, else
         # throws exception
         try:
             svnrepo.svnremoterepo(baseui, url).svn.uuid
         except Exception:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
             raise URLError(
                 '"%s" is not a valid Subversion source url.' % (url, ))
         return True
 
     def is_path_valid_repository(self, wire, path):
 
         # NOTE(marcink):  short circuit the check for SVN repo
         # the repos.open might be expensive to check, but we have one cheap
         # pre condition that we can use, to check for 'format' file
 
         if not os.path.isfile(os.path.join(path, 'format')):
             return False
 
         try:
             svn.repos.open(path)
         except svn.core.SubversionException:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
             return False
         return True
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         cmd = ['svnadmin', 'info', repo_path]
         stdout, stderr = subprocessio.run_command(cmd)
         return stdout
 
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         start_rev = None
         end_rev = None
         if start_ts:
             start_ts_svn = apr_time_t(start_ts)
             start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
         else:
             start_rev = 1
         if end_ts:
             end_ts_svn = apr_time_t(end_ts)
             end_rev = svn.repos.dated_revision(repo, end_ts_svn)
         else:
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
     def revision_properties(self, wire, revision):
-        repo = self._factory.repo(wire)
-        fs_ptr = svn.repos.fs(repo)
-        return svn.fs.revision_proplist(fs_ptr, revision)
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _revision_properties(_repo_id, _revision):
+            repo = self._factory.repo(wire)
+            fs_ptr = svn.repos.fs(repo)
+            return svn.fs.revision_proplist(fs_ptr, revision)
+        return _revision_properties(repo_id, revision)
 
     def revision_changes(self, wire, revision):
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         editor = svn.repos.ChangeCollector(fsobj, rev_root)
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         base_dir = ""
         send_deltas = False
         svn.repos.replay2(
             rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
             editor_ptr, editor_baton, None)
 
         added = []
         changed = []
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
         for path, change in editor.changes.iteritems():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
             if change.item_kind == svn.core.svn_node_dir:
                 continue
             if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
                                    svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(
                     "Action %s not supported on path %s" % (
                         change.action, path))
 
         changes = {
             'added': added,
             'changed': changed,
             'removed': removed,
         }
         return changes
 
+    @reraise_safe_exceptions
     def node_history(self, wire, path, revision, limit):
-        cross_copies = False
-        repo = self._factory.repo(wire)
-        fsobj = svn.repos.fs(repo)
-        rev_root = svn.fs.revision_root(fsobj, revision)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
+            cross_copies = False
+            repo = self._factory.repo(wire)
+            fsobj = svn.repos.fs(repo)
+            rev_root = svn.fs.revision_root(fsobj, revision)
 
-        history_revisions = []
-        history = svn.fs.node_history(rev_root, path)
-        history = svn.fs.history_prev(history, cross_copies)
-        while history:
-            __, node_revision = svn.fs.history_location(history)
-            history_revisions.append(node_revision)
-            if limit and len(history_revisions) >= limit:
-                break
+            history_revisions = []
+            history = svn.fs.node_history(rev_root, path)
             history = svn.fs.history_prev(history, cross_copies)
-        return history_revisions
+            while history:
+                __, node_revision = svn.fs.history_location(history)
+                history_revisions.append(node_revision)
+                if limit and len(history_revisions) >= limit:
+                    break
+                history = svn.fs.history_prev(history, cross_copies)
+            return history_revisions
+        return _assert_correct_path(context_uid, repo_id, path, revision, limit)
 
     def node_properties(self, wire, path, revision):
-        repo = self._factory.repo(wire)
-        fsobj = svn.repos.fs(repo)
-        rev_root = svn.fs.revision_root(fsobj, revision)
-        return svn.fs.node_proplist(rev_root, path)
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _node_properties(_repo_id, _path, _revision):
+            repo = self._factory.repo(wire)
+            fsobj = svn.repos.fs(repo)
+            rev_root = svn.fs.revision_root(fsobj, revision)
+            return svn.fs.node_proplist(rev_root, path)
+        return _node_properties(repo_id, path, revision)
 
     def file_annotate(self, wire, path, revision):
         abs_path = 'file://' + urllib.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
         start_rev = svn_opt_revision_value_t(0)
         peg_rev = svn_opt_revision_value_t(revision)
         end_rev = peg_rev
 
         annotations = []
 
         def receiver(line_no, revision, author, date, line, pool):
             annotations.append((line_no, revision, line))
 
         # TODO: Cannot use blame5, missing typemap function in the swig code
         try:
             svn.client.blame2(
                 file_uri, peg_rev, start_rev, end_rev,
                 receiver, svn.client.create_context())
         except svn.core.SubversionException as exc:
             log.exception("Error during blame operation.")
             raise Exception(
                 "Blame not supported or file does not exist at path %s. "
                 "Error %s." % (path, exc))
 
         return annotations
 
-    def get_node_type(self, wire, path, rev=None):
-        repo = self._factory.repo(wire)
-        fs_ptr = svn.repos.fs(repo)
-        if rev is None:
-            rev = svn.fs.youngest_rev(fs_ptr)
-        root = svn.fs.revision_root(fs_ptr, rev)
-        node = svn.fs.check_path(root, path)
-        return NODE_TYPE_MAPPING.get(node, None)
+    def get_node_type(self, wire, path, revision=None):
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_node_type(_repo_id, _path, _revision):
+            repo = self._factory.repo(wire)
+            fs_ptr = svn.repos.fs(repo)
+            if _revision is None:
+                _revision = svn.fs.youngest_rev(fs_ptr)
+            root = svn.fs.revision_root(fs_ptr, _revision)
+            node = svn.fs.check_path(root, path)
+            return NODE_TYPE_MAPPING.get(node, None)
+        return _get_node_type(repo_id, path, revision)
 
     def get_nodes(self, wire, path, revision=None):
-        repo = self._factory.repo(wire)
-        fsobj = svn.repos.fs(repo)
-        if revision is None:
-            revision = svn.fs.youngest_rev(fsobj)
-        root = svn.fs.revision_root(fsobj, revision)
-        entries = svn.fs.dir_entries(root, path)
-        result = []
-        for entry_path, entry_info in entries.iteritems():
-            result.append(
-                (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
-        return result
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_nodes(_repo_id, _path, _revision):
+            repo = self._factory.repo(wire)
+            fsobj = svn.repos.fs(repo)
+            if _revision is None:
+                _revision = svn.fs.youngest_rev(fsobj)
+            root = svn.fs.revision_root(fsobj, _revision)
+            entries = svn.fs.dir_entries(root, path)
+            result = []
+            for entry_path, entry_info in entries.iteritems():
+                result.append(
+                    (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
+            return result
+        return _get_nodes(repo_id, path, revision)
 
     def get_file_content(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
         return content.read()
 
     def get_file_size(self, wire, path, revision=None):
-        repo = self._factory.repo(wire)
-        fsobj = svn.repos.fs(repo)
-        if revision is None:
-            revision = svn.fs.youngest_revision(fsobj)
-        root = svn.fs.revision_root(fsobj, revision)
-        size = svn.fs.file_length(root, path)
-        return size
+
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _get_file_size(_repo_id, _path, _revision):
+            repo = self._factory.repo(wire)
+            fsobj = svn.repos.fs(repo)
+            if _revision is None:
+                _revision = svn.fs.youngest_revision(fsobj)
+            root = svn.fs.revision_root(fsobj, _revision)
+            size = svn.fs.file_length(root, path)
+            return size
+        return _get_file_size(repo_id, path, revision)
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
     def get_url_and_credentials(self, src_url):
         obj = urlparse.urlparse(src_url)
         username = obj.username or None
         password = obj.password or None
         return username, password, src_url
 
     def import_remote_repository(self, wire, src_url):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
         username, password, src_url = self.get_url_and_credentials(src_url)
         rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
                      '--trust-server-cert-failures=unknown-ca']
         if username and password:
             rdump_cmd += ['--username', username, '--password', password]
         rdump_cmd += [src_url]
 
         rdump = subprocess.Popen(
             rdump_cmd,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         load = subprocess.Popen(
             ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
 
         # TODO: johbo: This can be a very long operation, might be better
         # to track some kind of status and provide an api to check if the
         # import is done.
         rdump.wait()
         load.wait()
 
         log.debug('Return process ended with code: %s', rdump.returncode)
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
             log.error('svnrdump dump failed: statuscode %s: message: %s',
                       rdump.returncode, errors)
             reason = 'UNKNOWN'
             if 'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
 
             if reason == 'UNKNOWN':
                 reason = 'UNKNOWN:{}'.format(errors)
             raise Exception(
                 'Failed to dump the remote repository from %s. Reason:%s' % (
                     src_url, reason))
         if load.returncode != 0:
             raise Exception(
                 'Failed to load the dump of remote repository from %s.' %
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
         assert isinstance(message, str)
         assert isinstance(author, str)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
 
         rev = svn.fs.youngest_rev(fsobj)
         txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
         txn_root = svn.fs.txn_root(txn)
 
         for node in updated:
             TxnNodeProcessor(node, txn_root).update()
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp:
             apr_time = apr_time_t(timestamp)
             ts_formatted = svn.core.svn_time_to_cstring(apr_time)
             svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
 
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
         wire.update(cache=False)
         repo = self._factory.repo(wire)
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
             return diff_creator.generate_diff()
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
                 "Path might not exist %s, %s" % (path1, path2))
             return ""
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return False
 
     @reraise_safe_exceptions
+    def is_binary(self, wire, rev, path):
+        cache_on, context_uid, repo_id = self._cache_on(wire)
+
+        @self.region.conditional_cache_on_arguments(condition=cache_on)
+        def _is_binary(_repo_id, _rev, _path):
+            raw_bytes = self.get_file_content(wire, path, rev)
+            return raw_bytes and '\0' in raw_bytes
+
+        return _is_binary(repo_id, rev, path)
+
+    @reraise_safe_exceptions
     def run_svn_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         safe_call = False
         if '_safe' in opts:
             safe_call = True
 
         svnenv = os.environ.copy()
         svnenv.update(opts.pop('extra_env', {}))
 
         _opts = {'env': svnenv, 'shell': False}
 
         try:
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(p), ''.join(p.error)
         except (EnvironmentError, OSError) as err:
             cmd = ' '.join(cmd)  # human friendly CMD
             tb_err = ("Couldn't run svn command (%s).\n"
                       "Original error was:%s\n"
                       "Call options:%s\n"
                       % (cmd, err, _opts))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException()(tb_err)
 
     @reraise_safe_exceptions
     def install_hooks(self, wire, force=False):
         from vcsserver.hook_utils import install_svn_hooks
         repo_path = wire['path']
         binary_dir = settings.BINARY_DIR
         executable = None
         if binary_dir:
             executable = os.path.join(binary_dir, 'python')
         return install_svn_hooks(
             repo_path, executable=executable, force_create=force)
 
     @reraise_safe_exceptions
     def get_hooks_info(self, wire):
         from vcsserver.hook_utils import (
             get_svn_pre_hook_version, get_svn_post_hook_version)
         repo_path = wire['path']
         return {
             'pre_version': get_svn_pre_hook_version(repo_path),
             'post_version': get_svn_post_hook_version(repo_path),
         }
 
 
 class SvnDiffer(object):
     """
     Utility to create diffs based on difflib and the Subversion api
     """
 
     binary_content = False
 
     def __init__(
             self, repo, src_rev, src_path, tgt_rev, tgt_path,
             ignore_whitespace, context):
         self.repo = repo
         self.ignore_whitespace = ignore_whitespace
         self.context = context
 
         fsobj = svn.repos.fs(repo)
 
         self.tgt_rev = tgt_rev
         self.tgt_path = tgt_path or ''
         self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
         self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
 
         self.src_rev = src_rev
         self.src_path = src_path or self.tgt_path
         self.src_root = svn.fs.revision_root(fsobj, src_rev)
         self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
 
         self._validate()
 
     def _validate(self):
         if (self.tgt_kind != svn.core.svn_node_none and
                 self.src_kind != svn.core.svn_node_none and
                 self.src_kind != self.tgt_kind):
             # TODO: johbo: proper error handling
             raise Exception(
                 "Source and target are not compatible for diff generation. "
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
     def generate_diff(self):
         buf = StringIO.StringIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
     def _generate_dir_diff(self, buf):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
             self.src_root,
             self.src_path,
             '',  # src_entry
             self.tgt_root,
             self.tgt_path,
             editor_ptr, editor_baton,
             authorization_callback_allow_all,
             False,  # text_deltas
             svn.core.svn_depth_infinity,  # depth
             False,  # entry_props
             False,  # ignore_ancestry
         )
 
         for path, __, change in sorted(editor.changes):
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
     def _generate_file_diff(self, buf):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
         elif self.tgt_kind == svn.core.svn_node_none:
             change = "delete"
         tgt_base, tgt_path = vcspath.split(self.tgt_path)
         src_base, src_path = vcspath.split(self.src_path)
         self._generate_node_diff(
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
             self, buf, change, tgt_path, tgt_base, src_path, src_base):
 
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
             # we compare same revisions
             return
 
         tgt_full_path = vcspath.join(tgt_base, tgt_path)
         src_full_path = vcspath.join(src_base, src_path)
 
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
         if mime_type and not mime_type.startswith('text'):
             self.binary_content = True
             buf.write("=" * 67 + '\n')
             buf.write("Cannot display: file marked as a binary type.\n")
             buf.write("svn:mime-type = %s\n" % mime_type)
         buf.write("Index: %s\n" % (tgt_path, ))
         buf.write("=" * 67 + '\n')
         buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
             'tgt_path': tgt_path})
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
             buf.write("new file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- /dev/null\t(revision 0)\n")
             src_lines = []
         else:
             if change == 'delete':
                 buf.write("deleted file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- a/%s\t(revision %s)\n" % (
                 src_path, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
             buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
             tgt_lines = []
         else:
             buf.write("+++ b/%s\t(revision %s)\n" % (
                 tgt_path, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
             buf.writelines(udiff)
 
     def _get_mime_type(self, path):
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
         except svn.core.SubversionException:
             mime_type = svn.fs.node_prop(
                 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
         return mime_type
 
     def _svn_readlines(self, fs_root, node_path):
         if self.binary_content:
             return []
         node_kind = svn.fs.check_path(fs_root, node_path)
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
         content = svn.core.Stream(
             svn.fs.file_contents(fs_root, node_path)).read()
         return content.splitlines(True)
 
 
-
 class DiffChangeEditor(svn.delta.Editor):
     """
     Records changes between two given revisions
     """
 
     def __init__(self):
         self.changes = []
 
     def delete_entry(self, path, revision, parent_baton, pool=None):
         self.changes.append((path, None, 'delete'))
 
     def add_file(
             self, path, parent_baton, copyfrom_path, copyfrom_revision,
             file_pool=None):
         self.changes.append((path, 'file', 'add'))
 
     def open_file(self, path, parent_baton, base_revision, file_pool=None):
         self.changes.append((path, 'file', 'change'))
 
 
 def authorization_callback_allow_all(root, path, pool):
     return True
 
 
 class TxnNodeProcessor(object):
     """
     Utility to process the change of one node within a transaction root.
 
     It encapsulates the knowledge of how to add, update or remove
     a node for a given transaction root. The purpose is to support the method
     `SvnRemote.commit`.
     """
 
     def __init__(self, node, txn_root):
         assert isinstance(node['path'], str)
 
         self.node = node
         self.txn_root = txn_root
 
     def update(self):
         self._ensure_parent_dirs()
         self._add_file_if_node_does_not_exist()
         self._update_file_content()
         self._update_file_properties()
 
     def remove(self):
         svn.fs.delete(self.txn_root, self.node['path'])
         # TODO: Clean up directory if empty
 
     def _ensure_parent_dirs(self):
         curdir = vcspath.dirname(self.node['path'])
         dirs_to_create = []
         while not self._svn_path_exists(curdir):
             dirs_to_create.append(curdir)
             curdir = vcspath.dirname(curdir)
 
         for curdir in reversed(dirs_to_create):
             log.debug('Creating missing directory "%s"', curdir)
             svn.fs.make_dir(self.txn_root, curdir)
 
     def _svn_path_exists(self, path):
         path_status = svn.fs.check_path(self.txn_root, path)
         return path_status != svn.core.svn_node_none
 
     def _add_file_if_node_does_not_exist(self):
         kind = svn.fs.check_path(self.txn_root, self.node['path'])
         if kind == svn.core.svn_node_none:
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
         assert isinstance(self.node['content'], str)
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
         for key, value in properties.iteritems():
             svn.fs.change_node_prop(
                 self.txn_root, self.node['path'], key, value)
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
     return timestamp * 1E6
 
 
 def svn_opt_revision_value_t(num):
     """
     Put `num` into a `svn_opt_revision_value_t` structure.
     """
     value = svn.core.svn_opt_revision_value_t()
     value.number = num
     revision = svn.core.svn_opt_revision_t()
     revision.kind = svn.core.svn_opt_revision_number
     revision.value = value
     return revision

vcsserver/tests/test_git.py

@@ -1,165 +1,160 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import inspect
 
 import pytest
 import dulwich.errors
 from mock import Mock, patch
 
 from vcsserver import git
 
 
 SAMPLE_REFS = {
     'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
     'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
     'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
     'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
     'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
 }
 
 
 @pytest.fixture
 def git_remote():
     """
     A GitRemote instance with a mock factory.
     """
     factory = Mock()
     remote = git.GitRemote(factory)
     return remote
 
 
 def test_discover_git_version(git_remote):
     version = git_remote.discover_git_version()
     assert version
 
 
 class TestGitFetch(object):
     def setup(self):
         self.mock_repo = Mock()
         factory = Mock()
         factory.repo = Mock(return_value=self.mock_repo)
         self.remote_git = git.GitRemote(factory)
 
     def test_fetches_all_when_no_commit_ids_specified(self):
         def side_effect(determine_wants, *args, **kwargs):
             determine_wants(SAMPLE_REFS)
 
         with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
             mock_fetch.side_effect = side_effect
-            self.remote_git.pull(wire=None, url='/tmp/', apply_refs=False)
+            self.remote_git.pull(wire={}, url='/tmp/', apply_refs=False)
             determine_wants = self.mock_repo.object_store.determine_wants_all
             determine_wants.assert_called_once_with(SAMPLE_REFS)
 
     def test_fetches_specified_commits(self):
         selected_refs = {
             'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
             'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         def side_effect(determine_wants, *args, **kwargs):
             result = determine_wants(SAMPLE_REFS)
             assert sorted(result) == sorted(selected_refs.values())
             return result
 
         with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
             mock_fetch.side_effect = side_effect
             self.remote_git.pull(
-                wire=None, url='/tmp/', apply_refs=False,
+                wire={}, url='/tmp/', apply_refs=False,
                 refs=selected_refs.keys())
             determine_wants = self.mock_repo.object_store.determine_wants_all
             assert determine_wants.call_count == 0
 
     def test_get_remote_refs(self):
         factory = Mock()
         remote_git = git.GitRemote(factory)
         url = 'http://example.com/test/test.git'
         sample_refs = {
             'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
             'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
         }
 
         with patch('vcsserver.git.Repo', create=False) as mock_repo:
             mock_repo().get_refs.return_value = sample_refs
-            remote_refs = remote_git.get_remote_refs(wire=None, url=url)
+            remote_refs = remote_git.get_remote_refs(wire={}, url=url)
             mock_repo().get_refs.assert_called_once_with()
             assert remote_refs == sample_refs
 
-    def test_remove_ref(self):
-        ref_to_remove = 'refs/tags/v0.1.9'
-        self.mock_repo.refs = SAMPLE_REFS.copy()
-        self.remote_git.remove_ref(None, ref_to_remove)
-        assert ref_to_remove not in self.mock_repo.refs
-
 
 class TestReraiseSafeExceptions(object):
+
     def test_method_decorated_with_reraise_safe_exceptions(self):
         factory = Mock()
         git_remote = git.GitRemote(factory)
 
         def fake_function():
             return None
 
         decorator = git.reraise_safe_exceptions(fake_function)
 
         methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
         for method_name, method in methods:
             if not method_name.startswith('_'):
                 assert method.im_func.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
         (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
         (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
         (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
         (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
         (dulwich.errors.HangupException(), 'error'),
         (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
     ])
     def test_safe_exceptions_reraised(self, side_effect, expected_type):
         @git.reraise_safe_exceptions
         def fake_method():
             raise side_effect
 
         with pytest.raises(Exception) as exc_info:
             fake_method()
         assert type(exc_info.value) == Exception
         assert exc_info.value._vcs_kind == expected_type
 
 
 class TestDulwichRepoWrapper(object):
     def test_calls_close_on_delete(self):
         isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
         with isdir_patcher:
             repo = git.Repo('/tmp/abcde')
         with patch.object(git.DulwichRepo, 'close') as close_mock:
             del repo
         close_mock.assert_called_once_with()
 
 
 class TestGitFactory(object):
     def test_create_repo_returns_dulwich_wrapper(self):
 
         with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock:
             mock.side_effect = {'repo_objects': ''}
             factory = git.GitFactory()
             wire = {
                 'path': '/tmp/abcde'
             }
             isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
             with isdir_patcher:
                 result = factory._create_repo(wire, True)
             assert isinstance(result, git.Repo)

vcsserver/tests/test_hg.py

@@ -1,127 +1,108 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import inspect
 import sys
 import traceback
 
 import pytest
 from mercurial.error import LookupError
 from mock import Mock, MagicMock, patch
 
 from vcsserver import exceptions, hg, hgcompat
 
 
-class TestHGLookup(object):
-    def setup(self):
-        self.mock_repo = MagicMock()
-        self.mock_repo.__getitem__.side_effect = LookupError(
-            'revision_or_commit_id', 'index', 'message')
-        factory = Mock()
-        factory.repo = Mock(return_value=self.mock_repo)
-        self.remote_hg = hg.HgRemote(factory)
-
-    def test_fail_lookup_hg(self):
-        with pytest.raises(Exception) as exc_info:
-            self.remote_hg.lookup(
-                wire=None, revision='revision_or_commit_id', both=True)
-
-        assert exc_info.value._vcs_kind == 'lookup'
-        assert 'revision_or_commit_id' in exc_info.value.args
-
-
 class TestDiff(object):
     def test_raising_safe_exception_when_lookup_failed(self):
-        repo = Mock()
+
         factory = Mock()
-        factory.repo = Mock(return_value=repo)
         hg_remote = hg.HgRemote(factory)
         with patch('mercurial.patch.diff') as diff_mock:
             diff_mock.side_effect = LookupError(
                 'deadbeef', 'index', 'message')
             with pytest.raises(Exception) as exc_info:
                 hg_remote.diff(
-                    wire=None, rev1='deadbeef', rev2='deadbee1',
+                    wire={}, commit_id_1='deadbeef', commit_id_2='deadbee1',
                     file_filter=None, opt_git=True, opt_ignorews=True,
                     context=3)
             assert type(exc_info.value) == Exception
             assert exc_info.value._vcs_kind == 'lookup'
 
 
 class TestReraiseSafeExceptions(object):
     def test_method_decorated_with_reraise_safe_exceptions(self):
         factory = Mock()
         hg_remote = hg.HgRemote(factory)
         methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
         decorator = hg.reraise_safe_exceptions(None)
         for method_name, method in methods:
             if not method_name.startswith('_'):
                 assert method.im_func.__code__ == decorator.__code__
 
     @pytest.mark.parametrize('side_effect, expected_type', [
         (hgcompat.Abort(), 'abort'),
         (hgcompat.InterventionRequired(), 'abort'),
         (hgcompat.RepoLookupError(), 'lookup'),
         (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
         (hgcompat.RepoError(), 'error'),
         (hgcompat.RequirementError(), 'requirement'),
     ])
     def test_safe_exceptions_reraised(self, side_effect, expected_type):
         @hg.reraise_safe_exceptions
         def fake_method():
             raise side_effect
 
         with pytest.raises(Exception) as exc_info:
             fake_method()
         assert type(exc_info.value) == Exception
         assert exc_info.value._vcs_kind == expected_type
 
     def test_keeps_original_traceback(self):
         @hg.reraise_safe_exceptions
         def fake_method():
             try:
                 raise hgcompat.Abort()
             except:
                 self.original_traceback = traceback.format_tb(
                     sys.exc_info()[2])
                 raise
 
         try:
             fake_method()
         except Exception:
             new_traceback = traceback.format_tb(sys.exc_info()[2])
 
         new_traceback_tail = new_traceback[-len(self.original_traceback):]
         assert new_traceback_tail == self.original_traceback
 
     def test_maps_unknow_exceptions_to_unhandled(self):
         @hg.reraise_safe_exceptions
         def stub_method():
             raise ValueError('stub')
 
         with pytest.raises(Exception) as exc_info:
             stub_method()
         assert exc_info.value._vcs_kind == 'unhandled'
 
     def test_does_not_map_known_exceptions(self):
         @hg.reraise_safe_exceptions
         def stub_method():
             raise exceptions.LookupException()('stub')
 
         with pytest.raises(Exception) as exc_info:
             stub_method()
         assert exc_info.value._vcs_kind == 'lookup'

vcsserver/tests/test_svn.py

@@ -1,82 +1,87 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import mock
 import pytest
 import sys
 
 
 class MockPopen(object):
     def __init__(self, stderr):
         self.stdout = io.BytesIO('')
         self.stderr = io.BytesIO(stderr)
         self.returncode = 1
 
     def wait(self):
         pass
 
 
 INVALID_CERTIFICATE_STDERR = '\n'.join([
     'svnrdump: E230001: Unable to connect to a repository at URL url',
     'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
 ])
 
 
 @pytest.mark.parametrize('stderr,expected_reason', [
     (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
     ('svnrdump: E123456', 'UNKNOWN:svnrdump: E123456'),
 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
 @pytest.mark.xfail(sys.platform == "cygwin",
                    reason="SVN not packaged for Cygwin")
 def test_import_remote_repository_certificate_error(stderr, expected_reason):
     from vcsserver import svn
+    factory = mock.Mock()
+    factory.repo = mock.Mock(return_value=mock.Mock())
 
-    remote = svn.SvnRemote(None)
+    remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     with mock.patch('subprocess.Popen',
                     return_value=MockPopen(stderr)):
         with pytest.raises(Exception) as excinfo:
             remote.import_remote_repository({'path': 'path'}, 'url')
 
     expected_error_args = (
         'Failed to dump the remote repository from url. Reason:{}'.format(expected_reason),)
 
     assert excinfo.value.args == expected_error_args
 
 
 def test_svn_libraries_can_be_imported():
     import svn
     import svn.client
     assert svn.client is not None
 
 
 @pytest.mark.parametrize('example_url, parts', [
     ('http://server.com', (None, None, 'http://server.com')),
     ('http://[e-mail]', ('user', None, 'http://[e-mail]')),
     ('http://user:[e-mail]', ('user', 'pass', 'http://user:[e-mail]')),
     ('<script>', (None, None, '<script>')),
     ('http://', (None, None, 'http://')),
 ])
 def test_username_password_extraction_from_url(example_url, parts):
     from vcsserver import svn
 
-    remote = svn.SvnRemote(None)
+    factory = mock.Mock()
+    factory.repo = mock.Mock(return_value=mock.Mock())
+
+    remote = svn.SvnRemote(factory)
     remote.is_path_valid_repository = lambda wire, path: True
 
     assert remote.get_url_and_credentials(example_url) == parts

vcsserver/tweens/request_wrapper.py

@@ -1,58 +1,64 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
-
-
 import time
 import logging
 
-
+import vcsserver
 from vcsserver.utils import safe_str
 
 
 log = logging.getLogger(__name__)
 
 
 def get_access_path(request):
     environ = request.environ
     return environ.get('PATH_INFO')
 
 
+def get_user_agent(environ):
+    return environ.get('HTTP_USER_AGENT')
+
+
 class RequestWrapperTween(object):
     def __init__(self, handler, registry):
         self.handler = handler
         self.registry = registry
 
         # one-time configuration code goes here
 
     def __call__(self, request):
         start = time.time()
         try:
             response = self.handler(request)
         finally:
             end = time.time()
-
-            log.info('IP: %s Request to path: `%s` time: %.3fs',
-                     '127.0.0.1', safe_str(get_access_path(request)), end - start)
+            total = end - start
+            count = request.request_count()
+            _ver_ = vcsserver.__version__
+            log.info(
+                'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
+                count, '127.0.0.1', request.environ.get('REQUEST_METHOD'),
+                safe_str(get_access_path(request)), total, get_user_agent(request.environ), _ver_)
 
         return response
 
 
 def includeme(config):
     config.add_tween(
-        'vcsserver.tweens.RequestWrapperTween',
+        'vcsserver.tweens.request_wrapper.RequestWrapperTween',
     )