caches: added redis backend as an option

vcsserver/lib/rc_cache/__init__.py

@@ -1,60 +1,69 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 from dogpile.cache import register_backend
 
 register_backend(
     "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
+register_backend(
+    "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
+    "FileNamespaceBackend")
+
+register_backend(
+    "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
+    "RedisPickleBackend")
+
+
 log = logging.getLogger(__name__)
 
 from . import region_meta
-from .util import key_generator, get_default_cache_settings, make_region
+from .utils import (get_default_cache_settings, key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         avail_regions.add(namespace_name)
     log.debug('dogpile: found following cache regions: %s', avail_regions)
 
     # register them into namespace
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
             function_key_generator=key_generator
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
 
         log.debug('dogpile: registering a new region %s[%s]',
                   region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

vcsserver/lib/rc_cache/backends.py

@@ -1,51 +1,179 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import time
+import errno
 import logging
 
 from dogpile.cache.backends import memory as memory_backend
+from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import redis as redis_backend
+from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
+from dogpile.cache.util import memoized_property
+
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
     pickle_values = False
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
+
+
+class Serializer(object):
+    def _dumps(self, value, safe=False):
+        try:
+            return compat.pickle.dumps(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        try:
+            return compat.pickle.loads(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
+import fcntl
+flock_org = fcntl.flock
+
+
+class CustomLockFactory(FileLock):
+
+    pass
+
+
+class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+
+    def __init__(self, arguments):
+        arguments['lock_factory'] = CustomLockFactory
+        super(FileNamespaceBackend, self).__init__(arguments)
+
+    def list_keys(self, prefix=''):
+        def cond(v):
+            if not prefix:
+                return True
+
+            if v.startswith(prefix):
+                return True
+            return False
+
+        with self._dbm_file(True) as dbm:
+
+            return filter(cond, dbm.keys())
+
+    def get_store(self):
+        return self.filename
+
+    def get(self, key):
+        with self._dbm_file(False) as dbm:
+            if hasattr(dbm, 'get'):
+                value = dbm.get(key, NO_VALUE)
+            else:
+                # gdbm objects lack a .get method
+                try:
+                    value = dbm[key]
+                except KeyError:
+                    value = NO_VALUE
+            if value is not NO_VALUE:
+                value = self._loads(value)
+            return value
+
+    def set(self, key, value):
+        with self._dbm_file(True) as dbm:
+            dbm[key] = self._dumps(value)
+
+    def set_multi(self, mapping):
+        with self._dbm_file(True) as dbm:
+            for key, value in mapping.items():
+                dbm[key] = self._dumps(value)
+
+
+class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+    def list_keys(self, prefix=''):
+        if prefix:
+            prefix = prefix + '*'
+        return self.client.keys(prefix)
+
+    def get_store(self):
+        return self.client.connection_pool
+
+    def get(self, key):
+        value = self.client.get(key)
+        if value is None:
+            return NO_VALUE
+        return self._loads(value)
+
+    def set(self, key, value):
+        if self.redis_expiration_time:
+            self.client.setex(key, self.redis_expiration_time,
+                              self._dumps(value))
+        else:
+            self.client.set(key, self._dumps(value))
+
+    def set_multi(self, mapping):
+        mapping = dict(
+            (k, self._dumps(v))
+            for k, v in mapping.items()
+        )
+
+        if not self.redis_expiration_time:
+            self.client.mset(mapping)
+        else:
+            pipe = self.client.pipeline()
+            for key, value in mapping.items():
+                pipe.setex(key, self.redis_expiration_time, value)
+            pipe.execute()
+
+    def get_mutex(self, key):
+        u = redis_backend.u
+        if self.distributed_lock:
+            lock_key = u('_lock_{0}').format(key)
+            log.debug('Trying to acquire Redis lock for key %s', lock_key)
+            return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
+        else:
+            return None

vcsserver/lib/rc_cache/utils.py

@@ -1,136 +1,139 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import logging
 import functools
 
-from vcsserver.utils import safe_str, sha1
 from dogpile.cache import CacheRegion
 from dogpile.cache.util import compat
 
+from vcsserver.utils import safe_str, sha1
+
+
 log = logging.getLogger(__name__)
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=compat.string_type,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different than should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = compat.callable(expiration_time)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
         def decorator(fn):
             if to_str is compat.string_type:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, fn)
             else:
                 key_generator = function_key_generator(namespace, fn, to_str=to_str)
 
             @functools.wraps(fn)
             def decorate(*arg, **kw):
                 key = key_generator(*arg, **kw)
 
                 @functools.wraps(fn)
                 def creator():
                     return fn(*arg, **kw)
 
                 if not condition:
                     return creator()
 
                 timeout = expiration_time() if expiration_time_is_callable \
                     else expiration_time
 
                 return self.get_or_create(key, creator, timeout, should_cache_fn)
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
             def refresh(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 value = fn(*arg, **kw)
                 self.set(key, value)
                 return value
 
             decorate.set = set_
             decorate.invalidate = invalidate
             decorate.refresh = refresh
             decorate.get = get
             decorate.original = fn
             decorate.key_generator = key_generator
+            decorate.__wrapped__ = fn
 
             return decorate
 
         return decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
-                if isinstance(val, basestring):
+                if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
 def key_generator(namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
         namespace_pref = namespace or 'default'
         arg_key = compute_key_from_params(*args)
         final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key