merged stable into default

.release.cfg

@@ -0,0 +1,16 b''
+[DEFAULT]
+done = false
+
+[task:bump_version]
+done = true
+
+[task:fixes_on_stable]
+done = true
+
+[task:pip2nix_generated]
+done = true
+
+[release]
+state = prepared
+version = 4.25.2
+

vcsserver/lib/_vendor/redis_lock/__init__.py

@@ -0,0 +1,390 b''
+import sys
+import threading
+import weakref
+from base64 import b64encode
+from logging import getLogger
+from os import urandom
+
+from redis import StrictRedis
+
+__version__ = '3.7.0'
+
+loggers = {
+    k: getLogger("vcsserver." + ".".join((__name__, k)))
+    for k in [
+        "acquire",
+        "refresh.thread.start",
+        "refresh.thread.stop",
+        "refresh.thread.exit",
+        "refresh.start",
+        "refresh.shutdown",
+        "refresh.exit",
+        "release",
+    ]
+}
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    text_type = str
+    binary_type = bytes
+else:
+    text_type = unicode  # noqa
+    binary_type = str
+
+
+# Check if the id match. If not, return an error code.
+UNLOCK_SCRIPT = b"""
+    if redis.call("get", KEYS[1]) ~= ARGV[1] then
+        return 1
+    else
+        redis.call("del", KEYS[2])
+        redis.call("lpush", KEYS[2], 1)
+        redis.call("pexpire", KEYS[2], ARGV[2])
+        redis.call("del", KEYS[1])
+        return 0
+    end
+"""
+
+# Covers both cases when key doesn't exist and doesn't equal to lock's id
+EXTEND_SCRIPT = b"""
+    if redis.call("get", KEYS[1]) ~= ARGV[1] then
+        return 1
+    elseif redis.call("ttl", KEYS[1]) < 0 then
+        return 2
+    else
+        redis.call("expire", KEYS[1], ARGV[2])
+        return 0
+    end
+"""
+
+RESET_SCRIPT = b"""
+    redis.call('del', KEYS[2])
+    redis.call('lpush', KEYS[2], 1)
+    redis.call('pexpire', KEYS[2], ARGV[2])
+    return redis.call('del', KEYS[1])
+"""
+
+RESET_ALL_SCRIPT = b"""
+    local locks = redis.call('keys', 'lock:*')
+    local signal
+    for _, lock in pairs(locks) do
+        signal = 'lock-signal:' .. string.sub(lock, 6)
+        redis.call('del', signal)
+        redis.call('lpush', signal, 1)
+        redis.call('expire', signal, 1)
+        redis.call('del', lock)
+    end
+    return #locks
+"""
+
+
+class AlreadyAcquired(RuntimeError):
+    pass
+
+
+class NotAcquired(RuntimeError):
+    pass
+
+
+class AlreadyStarted(RuntimeError):
+    pass
+
+
+class TimeoutNotUsable(RuntimeError):
+    pass
+
+
+class InvalidTimeout(RuntimeError):
+    pass
+
+
+class TimeoutTooLarge(RuntimeError):
+    pass
+
+
+class NotExpirable(RuntimeError):
+    pass
+
+
+class Lock(object):
+    """
+    A Lock context manager implemented via redis SETNX/BLPOP.
+    """
+    unlock_script = None
+    extend_script = None
+    reset_script = None
+    reset_all_script = None
+
+    def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
+        """
+        :param redis_client:
+            An instance of :class:`~StrictRedis`.
+        :param name:
+            The name (redis key) the lock should have.
+        :param expire:
+            The lock expiry time in seconds. If left at the default (None)
+            the lock will not expire.
+        :param id:
+            The ID (redis value) the lock should have. A random value is
+            generated when left at the default.
+
+            Note that if you specify this then the lock is marked as "held". Acquires
+            won't be possible.
+        :param auto_renewal:
+            If set to ``True``, Lock will automatically renew the lock so that it
+            doesn't expire for as long as the lock is held (acquire() called
+            or running in a context manager).
+
+            Implementation note: Renewal will happen using a daemon thread with
+            an interval of ``expire*2/3``. If wishing to use a different renewal
+            time, subclass Lock, call ``super().__init__()`` then set
+            ``self._lock_renewal_interval`` to your desired interval.
+        :param strict:
+            If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
+        :param signal_expire:
+            Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
+        """
+        if strict and not isinstance(redis_client, StrictRedis):
+            raise ValueError("redis_client must be instance of StrictRedis. "
+                             "Use strict=False if you know what you're doing.")
+        if auto_renewal and expire is None:
+            raise ValueError("Expire may not be None when auto_renewal is set")
+
+        self._client = redis_client
+
+        if expire:
+            expire = int(expire)
+            if expire < 0:
+                raise ValueError("A negative expire is not acceptable.")
+        else:
+            expire = None
+        self._expire = expire
+
+        self._signal_expire = signal_expire
+        if id is None:
+            self._id = b64encode(urandom(18)).decode('ascii')
+        elif isinstance(id, binary_type):
+            try:
+                self._id = id.decode('ascii')
+            except UnicodeDecodeError:
+                self._id = b64encode(id).decode('ascii')
+        elif isinstance(id, text_type):
+            self._id = id
+        else:
+            raise TypeError("Incorrect type for `id`. Must be bytes/str not %s." % type(id))
+        self._name = 'lock:' + name
+        self._signal = 'lock-signal:' + name
+        self._lock_renewal_interval = (float(expire) * 2 / 3
+                                       if auto_renewal
+                                       else None)
+        self._lock_renewal_thread = None
+
+        self.register_scripts(redis_client)
+
+    @classmethod
+    def register_scripts(cls, redis_client):
+        global reset_all_script
+        if reset_all_script is None:
+            reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
+            cls.unlock_script = redis_client.register_script(UNLOCK_SCRIPT)
+            cls.extend_script = redis_client.register_script(EXTEND_SCRIPT)
+            cls.reset_script = redis_client.register_script(RESET_SCRIPT)
+            cls.reset_all_script = redis_client.register_script(RESET_ALL_SCRIPT)
+
+    @property
+    def _held(self):
+        return self.id == self.get_owner_id()
+
+    def reset(self):
+        """
+        Forcibly deletes the lock. Use this with care.
+        """
+        self.reset_script(client=self._client, keys=(self._name, self._signal), args=(self.id, self._signal_expire))
+
+    @property
+    def id(self):
+        return self._id
+
+    def get_owner_id(self):
+        owner_id = self._client.get(self._name)
+        if isinstance(owner_id, binary_type):
+            owner_id = owner_id.decode('ascii', 'replace')
+        return owner_id
+
+    def acquire(self, blocking=True, timeout=None):
+        """
+        :param blocking:
+            Boolean value specifying whether lock should be blocking or not.
+        :param timeout:
+            An integer value specifying the maximum number of seconds to block.
+        """
+        logger = loggers["acquire"]
+
+        logger.debug("Getting acquire on %r ...", self._name)
+
+        if self._held:
+            owner_id = self.get_owner_id()
+            raise AlreadyAcquired("Already acquired from this Lock instance. Lock id: {}".format(owner_id))
+
+        if not blocking and timeout is not None:
+            raise TimeoutNotUsable("Timeout cannot be used if blocking=False")
+
+        if timeout:
+            timeout = int(timeout)
+            if timeout < 0:
+                raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)
+
+            if self._expire and not self._lock_renewal_interval and timeout > self._expire:
+                raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))
+
+        busy = True
+        blpop_timeout = timeout or self._expire or 0
+        timed_out = False
+        while busy:
+            busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
+            if busy:
+                if timed_out:
+                    return False
+                elif blocking:
+                    timed_out = not self._client.blpop(self._signal, blpop_timeout) and timeout
+                else:
+                    logger.warning("Failed to get %r.", self._name)
+                    return False
+
+        logger.info("Got lock for %r.", self._name)
+        if self._lock_renewal_interval is not None:
+            self._start_lock_renewer()
+        return True
+
+    def extend(self, expire=None):
+        """Extends expiration time of the lock.
+
+        :param expire:
+            New expiration time. If ``None`` - `expire` provided during
+            lock initialization will be taken.
+        """
+        if expire:
+            expire = int(expire)
+            if expire < 0:
+                raise ValueError("A negative expire is not acceptable.")
+        elif self._expire is not None:
+            expire = self._expire
+        else:
+            raise TypeError(
+                "To extend a lock 'expire' must be provided as an "
+                "argument to extend() method or at initialization time."
+            )
+
+        error = self.extend_script(client=self._client, keys=(self._name, self._signal), args=(self._id, expire))
+        if error == 1:
+            raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
+        elif error == 2:
+            raise NotExpirable("Lock %s has no assigned expiration time" % self._name)
+        elif error:
+            raise RuntimeError("Unsupported error code %s from EXTEND script" % error)
+
+    @staticmethod
+    def _lock_renewer(lockref, interval, stop):
+        """
+        Renew the lock key in redis every `interval` seconds for as long
+        as `self._lock_renewal_thread.should_exit` is False.
+        """
+        while not stop.wait(timeout=interval):
+            loggers["refresh.thread.start"].debug("Refreshing lock")
+            lock = lockref()
+            if lock is None:
+                loggers["refresh.thread.stop"].debug(
+                    "The lock no longer exists, stopping lock refreshing"
+                )
+                break
+            lock.extend(expire=lock._expire)
+            del lock
+        loggers["refresh.thread.exit"].debug("Exit requested, stopping lock refreshing")
+
+    def _start_lock_renewer(self):
+        """
+        Starts the lock refresher thread.
+        """
+        if self._lock_renewal_thread is not None:
+            raise AlreadyStarted("Lock refresh thread already started")
+
+        loggers["refresh.start"].debug(
+            "Starting thread to refresh lock every %s seconds",
+            self._lock_renewal_interval
+        )
+        self._lock_renewal_stop = threading.Event()
+        self._lock_renewal_thread = threading.Thread(
+            group=None,
+            target=self._lock_renewer,
+            kwargs={'lockref': weakref.ref(self),
+                    'interval': self._lock_renewal_interval,
+                    'stop': self._lock_renewal_stop}
+        )
+        self._lock_renewal_thread.setDaemon(True)
+        self._lock_renewal_thread.start()
+
+    def _stop_lock_renewer(self):
+        """
+        Stop the lock renewer.
+
+        This signals the renewal thread and waits for its exit.
+        """
+        if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
+            return
+        loggers["refresh.shutdown"].debug("Signalling the lock refresher to stop")
+        self._lock_renewal_stop.set()
+        self._lock_renewal_thread.join()
+        self._lock_renewal_thread = None
+        loggers["refresh.exit"].debug("Lock refresher has stopped")
+
+    def __enter__(self):
+        acquired = self.acquire(blocking=True)
+        assert acquired, "Lock wasn't acquired, but blocking=True"
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None):
+        self.release()
+
+    def release(self):
+        """Releases the lock, that was acquired with the same object.
+
+        .. note::
+
+            If you want to release a lock that you acquired in a different place you have two choices:
+
+            * Use ``Lock("name", id=id_from_other_place).release()``
+            * Use ``Lock("name").reset()``
+        """
+        if self._lock_renewal_thread is not None:
+            self._stop_lock_renewer()
+        loggers["release"].debug("Releasing %r.", self._name)
+        error = self.unlock_script(client=self._client, keys=(self._name, self._signal), args=(self._id, self._signal_expire))
+        if error == 1:
+            raise NotAcquired("Lock %s is not acquired or it already expired." % self._name)
+        elif error:
+            raise RuntimeError("Unsupported error code %s from EXTEND script." % error)
+
+    def locked(self):
+        """
+        Return true if the lock is acquired.
+
+        Checks that lock with same name already exists. This method returns true, even if
+        lock have another id.
+        """
+        return self._client.exists(self._name) == 1
+
+
+reset_all_script = None
+
+
+def reset_all(redis_client):
+    """
+    Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
+
+    :param redis_client:
+        An instance of :class:`~StrictRedis`.
+    """
+    Lock.register_scripts(redis_client)
+
+    reset_all_script(client=redis_client)  # noqa

.bumpversion.cfg

@@ -1,5 +1,6 b''
 [bumpversion]
-current_version = 4.25.0
+current_version = 4.25.2
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]
+

.hgtags

@@ -74,3 +74,6 b' 179d989bcfe02c6227f9f6aa9236cbbe1c14c400'
 383aee8b1652affaa26aefe336a89ee366b2b26d v4.23.2
 bc1a8141cc51fc23c455ebc50c6609c810b46f8d v4.24.0
 530a1c03caabc806ea1ef34605f8f67f18c70e55 v4.24.1
+5908ae65cee1043982e1b26d7b618af5fcfebbb3 v4.25.0
+cce8bcdf75090d5943a1e9706fe5212d7b5d1fa1 v4.25.1
+8610c4bf846c63bbc95d3ddfb53fadaaa9c7aa42 v4.25.2

pkgs/python-packages.nix

@@ -784,7 +784,7 b' self: super: {'
     };
   };
   "rhodecode-vcsserver" = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.25.0";
+    name = "rhodecode-vcsserver-4.25.2";
     buildInputs = [
       self."pytest"
       self."py"

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.25.0
  No newline at end of file
+4.25.2
  No newline at end of file

vcsserver/hg.py

@@ -609,6 +609,7 b' class HgRemote(RemoteBase):'
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         cache_on, context_uid, repo_id = self._cache_on(wire)
+
         @self.region.conditional_cache_on_arguments(condition=cache_on)
         def _lookup(_context_uid, _repo_id, _revision, _both):
 

vcsserver/hgcompat.py

@@ -67,7 +67,7 b' from mercurial.url import httpbasicauthh'
 def get_ctx(repo, ref):
     try:
         ctx = repo[ref]
-    except ProgrammingError:
+    except (ProgrammingError, TypeError):
         # we're unable to find the rev using a regular lookup, we fallback
         # to slower, but backward compat revsymbol usage
         ctx = revsymbol(repo, ref)

vcsserver/http_main.py

@@ -433,7 +433,7 b' class HTTPApplication(object):'
                     should_store_exc = False
 
             if should_store_exc:
-                store_exception(id(exc_info), exc_info)
+                store_exception(id(exc_info), exc_info, request_path=request.path)
 
             tb_info = ''.join(
                 traceback.format_exception(exc_type, exc_value, exc_traceback))
@@ -452,6 +452,7 b' class HTTPApplication(object):'
                     'type': type_
                 }
             }
+
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:

vcsserver/lib/exc_tracking.py

@@ -68,7 +68,7 b' def get_exc_store():'
     return _exc_store_path
 
 
-def _store_exception(exc_id, exc_info, prefix):
+def _store_exception(exc_id, exc_info, prefix, request_path=''):
     exc_type, exc_value, exc_traceback = exc_info
 
     tb = ''.join(traceback.format_exception(
@@ -101,8 +101,13 b' def _store_exception(exc_id, exc_info, p'
         f.write(exc_data)
     log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
 
+    log.error(
+        'error occurred handling this request.\n'
+        'Path: `%s`, tb: %s',
+        request_path, tb)
 
-def store_exception(exc_id, exc_info, prefix=global_prefix):
+
+def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
     """
     Example usage::
 
@@ -111,7 +116,8 b' def store_exception(exc_id, exc_info, pr'
     """
 
     try:
-        _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
+        _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
+                         request_path=request_path)
     except Exception:
         log.exception('Failed to store exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.

vcsserver/lib/rc_cache/backends.py

@@ -124,7 +124,14 b' class FileNamespaceBackend(PickleSeriali'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
-        super(FileNamespaceBackend, self).__init__(arguments)
+        db_file = arguments.get('filename')
+
+        log.debug('initialing %s DB in %s', self.__class__.__name__, db_file)
+        try:
+            super(FileNamespaceBackend, self).__init__(arguments)
+        except Exception:
+            log.error('Failed to initialize db at: %s', db_file)
+            raise
 
     def __repr__(self):
         return '{} `{}`'.format(self.__class__, self.filename)
@@ -141,13 +148,16 b' class FileNamespaceBackend(PickleSeriali'
             return False
 
         with self._dbm_file(True) as dbm:
-
-            return filter(cond, dbm.keys())
+            try:
+                return filter(cond, dbm.keys())
+            except Exception:
+                log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
+                raise
 
     def get_store(self):
         return self.filename
 
-    def get(self, key):
+    def _dbm_get(self, key):
         with self._dbm_file(False) as dbm:
             if hasattr(dbm, 'get'):
                 value = dbm.get(key, NO_VALUE)
@@ -161,6 +171,13 b' class FileNamespaceBackend(PickleSeriali'
                 value = self._loads(value)
             return value
 
+    def get(self, key):
+        try:
+            return self._dbm_get(key)
+        except Exception:
+            log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
+            raise
+
     def set(self, key, value):
         with self._dbm_file(True) as dbm:
             dbm[key] = self._dumps(value)
@@ -234,11 +251,17 b' class BaseRedisBackend(redis_backend.Red'
             pipe.execute()
 
     def get_mutex(self, key):
-        u = redis_backend.u
         if self.distributed_lock:
-            lock_key = u('_lock_{0}').format(key)
+            lock_key = redis_backend.u('_lock_{0}').format(key)
             log.debug('Trying to acquire Redis lock for key %s', lock_key)
-            return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
+
+            auto_renewal = True
+            lock_timeout = self.lock_timeout
+            if auto_renewal and not self.lock_timeout:
+                # set default timeout for auto_renewal
+                lock_timeout = 10
+            return get_mutex_lock(self.client, lock_key, lock_timeout,
+                                  auto_renewal=auto_renewal)
         else:
             return None
 
@@ -251,3 +274,34 b' class RedisPickleBackend(PickleSerialize'
 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
     key_prefix = 'redis_msgpack_backend'
     pass
+
+
+def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
+    import redis_lock
+
+    class _RedisLockWrapper(object):
+        """LockWrapper for redis_lock"""
+
+        def __init__(self):
+            pass
+
+        @property
+        def lock(self):
+            return redis_lock.Lock(
+                redis_client=client,
+                name=lock_key,
+                expire=lock_timeout,
+                auto_renewal=auto_renewal,
+                strict=True,
+            )
+
+        def acquire(self, wait=True):
+            return self.lock.acquire(wait)
+
+        def release(self):
+            try:
+                self.lock.release()
+            except redis_lock.NotAcquired:
+                pass
+
+    return _RedisLockWrapper()