caches: don't use key_manglers instead prefix keys based on backend.

vcsserver/lib/rc_cache/__init__.py

@@ -34,7 +34,7 b' register_backend('
 log = logging.getLogger(__name__)
 
 from . import region_meta
-from .utils import (get_default_cache_settings, key_generator, make_region)
+from .utils import (get_default_cache_settings, backend_key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):
@@ -55,13 +55,12 b' def configure_dogpile_cache(settings):'
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
-            function_key_generator=key_generator
+            function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
-
-        log.debug('dogpile: registering a new region %s[%s]',
-                  region_name, new_region.__dict__)
+        new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
+        log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 

vcsserver/lib/rc_cache/backends.py

@@ -19,6 +19,8 b' import time'
 import errno
 import logging
 
+import msgpack
+from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import redis as redis_backend
@@ -34,6 +36,7 b' log = logging.getLogger(__name__)'
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
+    key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
@@ -58,7 +61,8 b' class LRUMemoryBackend(memory_backend.Me'
             self.delete(key)
 
 
-class Serializer(object):
+class PickleSerializer(object):
+
     def _dumps(self, value, safe=False):
         try:
             return compat.pickle.dumps(value)
@@ -78,6 +82,32 b' class Serializer(object):'
                 raise
 
 
+class MsgPackSerializer(object):
+
+    def _dumps(self, value, safe=False):
+        try:
+            return msgpack.packb(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        """
+        pickle maintained the `CachedValue` wrapper of the tuple
+        msgpack does not, so it must be added back in.
+       """
+        try:
+            value = msgpack.unpackb(value, use_list=False)
+            return CachedValue(*value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
 import fcntl
 flock_org = fcntl.flock
 
@@ -87,13 +117,16 b' class CustomLockFactory(FileLock):'
     pass
 
 
-class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
+    key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         super(FileNamespaceBackend, self).__init__(arguments)
 
     def list_keys(self, prefix=''):
+        prefix = '{}:{}'.format(self.key_prefix, prefix)
+
         def cond(v):
             if not prefix:
                 return True
@@ -133,10 +166,9 b' class FileNamespaceBackend(Serializer, f'
                 dbm[key] = self._dumps(value)
 
 
-class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+class BaseRedisBackend(redis_backend.RedisBackend):
     def list_keys(self, prefix=''):
-        if prefix:
-            prefix = prefix + '*'
+        prefix = '{}:{}*'.format(self.key_prefix, prefix)
         return self.client.keys(prefix)
 
     def get_store(self):
@@ -148,6 +180,15 b' class RedisPickleBackend(Serializer, red'
             return NO_VALUE
         return self._loads(value)
 
+    def get_multi(self, keys):
+        if not keys:
+            return []
+        values = self.client.mget(keys)
+        loads = self._loads
+        return [
+            loads(v) if v is not None else NO_VALUE
+            for v in values]
+
     def set(self, key, value):
         if self.redis_expiration_time:
             self.client.setex(key, self.redis_expiration_time,
@@ -156,8 +197,9 b' class RedisPickleBackend(Serializer, red'
             self.client.set(key, self._dumps(value))
 
     def set_multi(self, mapping):
+        dumps = self._dumps
         mapping = dict(
-            (k, self._dumps(v))
+            (k, dumps(v))
             for k, v in mapping.items()
         )
 
@@ -177,3 +219,13 b' class RedisPickleBackend(Serializer, red'
             return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
         else:
             return None
+
+
+class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
+    key_prefix = 'redis_pickle_backend'
+    pass
+
+
+class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
+    key_prefix = 'redis_msgpack_backend'
+    pass

vcsserver/lib/rc_cache/utils.py

@@ -126,13 +126,23 b' def compute_key_from_params(*args):'
     return sha1("_".join(map(safe_str, args)))
 
 
-def key_generator(namespace, fn):
+def backend_key_generator(backend):
+    """
+    Special wrapper that also sends over the backend to the key generator
+    """
+    def wrapper(namespace, fn):
+        return key_generator(backend, namespace, fn)
+    return wrapper
+
+
+def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
-        namespace_pref = namespace or 'default'
+        backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
+        namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
-        final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+        final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key