caches: don't use key_manglers instead prefix keys based on backend.

vcsserver/lib/rc_cache/__init__.py

@@ -1,69 +1,68 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 from dogpile.cache import register_backend
 
 register_backend(
     "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
 register_backend(
     "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
     "FileNamespaceBackend")
 
 register_backend(
     "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
     "RedisPickleBackend")
 
 
 log = logging.getLogger(__name__)
 
 from . import region_meta
-from .utils import (get_default_cache_settings, key_generator, make_region)
+from .utils import (get_default_cache_settings, backend_key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         avail_regions.add(namespace_name)
     log.debug('dogpile: found following cache regions: %s', avail_regions)
 
     # register them into namespace
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
-            function_key_generator=key_generator
+            function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
-
-        log.debug('dogpile: registering a new region %s[%s]',
-                  region_name, new_region.__dict__)
+        new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
+        log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

vcsserver/lib/rc_cache/backends.py

@@ -1,179 +1,231 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import time
 import errno
 import logging
 
+import msgpack
+from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import redis as redis_backend
 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
 from dogpile.cache.util import memoized_property
 
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
+    key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
 
 
-class Serializer(object):
+class PickleSerializer(object):
+
     def _dumps(self, value, safe=False):
         try:
             return compat.pickle.dumps(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
     def _loads(self, value, safe=True):
         try:
             return compat.pickle.loads(value)
         except Exception:
             if safe:
                 return NO_VALUE
             else:
                 raise
 
 
+class MsgPackSerializer(object):
+
+    def _dumps(self, value, safe=False):
+        try:
+            return msgpack.packb(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        """
+        pickle maintained the `CachedValue` wrapper of the tuple
+        msgpack does not, so it must be added back in.
+       """
+        try:
+            value = msgpack.unpackb(value, use_list=False)
+            return CachedValue(*value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
 import fcntl
 flock_org = fcntl.flock
 
 
 class CustomLockFactory(FileLock):
 
     pass
 
 
-class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
+    key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         super(FileNamespaceBackend, self).__init__(arguments)
 
     def list_keys(self, prefix=''):
+        prefix = '{}:{}'.format(self.key_prefix, prefix)
+
         def cond(v):
             if not prefix:
                 return True
 
             if v.startswith(prefix):
                 return True
             return False
 
         with self._dbm_file(True) as dbm:
 
             return filter(cond, dbm.keys())
 
     def get_store(self):
         return self.filename
 
     def get(self, key):
         with self._dbm_file(False) as dbm:
             if hasattr(dbm, 'get'):
                 value = dbm.get(key, NO_VALUE)
             else:
                 # gdbm objects lack a .get method
                 try:
                     value = dbm[key]
                 except KeyError:
                     value = NO_VALUE
             if value is not NO_VALUE:
                 value = self._loads(value)
             return value
 
     def set(self, key, value):
         with self._dbm_file(True) as dbm:
             dbm[key] = self._dumps(value)
 
     def set_multi(self, mapping):
         with self._dbm_file(True) as dbm:
             for key, value in mapping.items():
                 dbm[key] = self._dumps(value)
 
 
-class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+class BaseRedisBackend(redis_backend.RedisBackend):
     def list_keys(self, prefix=''):
-        if prefix:
-            prefix = prefix + '*'
+        prefix = '{}:{}*'.format(self.key_prefix, prefix)
         return self.client.keys(prefix)
 
     def get_store(self):
         return self.client.connection_pool
 
     def get(self, key):
         value = self.client.get(key)
         if value is None:
             return NO_VALUE
         return self._loads(value)
 
+    def get_multi(self, keys):
+        if not keys:
+            return []
+        values = self.client.mget(keys)
+        loads = self._loads
+        return [
+            loads(v) if v is not None else NO_VALUE
+            for v in values]
+
     def set(self, key, value):
         if self.redis_expiration_time:
             self.client.setex(key, self.redis_expiration_time,
                               self._dumps(value))
         else:
             self.client.set(key, self._dumps(value))
 
     def set_multi(self, mapping):
+        dumps = self._dumps
         mapping = dict(
-            (k, self._dumps(v))
+            (k, dumps(v))
             for k, v in mapping.items()
         )
 
         if not self.redis_expiration_time:
             self.client.mset(mapping)
         else:
             pipe = self.client.pipeline()
             for key, value in mapping.items():
                 pipe.setex(key, self.redis_expiration_time, value)
             pipe.execute()
 
     def get_mutex(self, key):
         u = redis_backend.u
         if self.distributed_lock:
             lock_key = u('_lock_{0}').format(key)
             log.debug('Trying to acquire Redis lock for key %s', lock_key)
             return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
         else:
             return None
+
+
+class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
+    key_prefix = 'redis_pickle_backend'
+    pass
+
+
+class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
+    key_prefix = 'redis_msgpack_backend'
+    pass

vcsserver/lib/rc_cache/utils.py

@@ -1,139 +1,149 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import logging
 import functools
 
 from dogpile.cache import CacheRegion
 from dogpile.cache.util import compat
 
 from vcsserver.utils import safe_str, sha1
 
 
 log = logging.getLogger(__name__)
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=compat.string_type,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different than should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = compat.callable(expiration_time)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
         def decorator(fn):
             if to_str is compat.string_type:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, fn)
             else:
                 key_generator = function_key_generator(namespace, fn, to_str=to_str)
 
             @functools.wraps(fn)
             def decorate(*arg, **kw):
                 key = key_generator(*arg, **kw)
 
                 @functools.wraps(fn)
                 def creator():
                     return fn(*arg, **kw)
 
                 if not condition:
                     return creator()
 
                 timeout = expiration_time() if expiration_time_is_callable \
                     else expiration_time
 
                 return self.get_or_create(key, creator, timeout, should_cache_fn)
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
             def refresh(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 value = fn(*arg, **kw)
                 self.set(key, value)
                 return value
 
             decorate.set = set_
             decorate.invalidate = invalidate
             decorate.refresh = refresh
             decorate.get = get
             decorate.original = fn
             decorate.key_generator = key_generator
             decorate.__wrapped__ = fn
 
             return decorate
 
         return decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
                 if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
-def key_generator(namespace, fn):
+def backend_key_generator(backend):
+    """
+    Special wrapper that also sends over the backend to the key generator
+    """
+    def wrapper(namespace, fn):
+        return key_generator(backend, namespace, fn)
+    return wrapper
+
+
+def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
-        namespace_pref = namespace or 'default'
+        backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
+        namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
-        final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+        final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key