diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py --- a/vcsserver/lib/rc_cache/__init__.py +++ b/vcsserver/lib/rc_cache/__init__.py @@ -34,7 +34,7 @@ register_backend( log = logging.getLogger(__name__) from . import region_meta -from .utils import (get_default_cache_settings, key_generator, make_region) +from .utils import (get_default_cache_settings, backend_key_generator, make_region) def configure_dogpile_cache(settings): @@ -55,13 +55,12 @@ def configure_dogpile_cache(settings): for region_name in avail_regions: new_region = make_region( name=region_name, - function_key_generator=key_generator + function_key_generator=None ) new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) - - log.debug('dogpile: registering a new region %s[%s]', - region_name, new_region.__dict__) + new_region.function_key_generator = backend_key_generator(new_region.actual_backend) + log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__) region_meta.dogpile_cache_regions[region_name] = new_region diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py --- a/vcsserver/lib/rc_cache/backends.py +++ b/vcsserver/lib/rc_cache/backends.py @@ -19,6 +19,8 @@ import time import errno import logging +import msgpack +from dogpile.cache.api import CachedValue from dogpile.cache.backends import memory as memory_backend from dogpile.cache.backends import file as file_backend from dogpile.cache.backends import redis as redis_backend @@ -34,6 +36,7 @@ log = logging.getLogger(__name__) class LRUMemoryBackend(memory_backend.MemoryBackend): + key_prefix = 'lru_mem_backend' pickle_values = False def __init__(self, arguments): @@ -58,7 +61,8 @@ class LRUMemoryBackend(memory_backend.Me self.delete(key) -class Serializer(object): +class PickleSerializer(object): + def _dumps(self, value, safe=False): try: return compat.pickle.dumps(value) @@ -78,6 +82,32 @@ class Serializer(object): raise +class MsgPackSerializer(object): + + def _dumps(self, value, safe=False): + try: + return msgpack.packb(value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + def _loads(self, value, safe=True): + """ + pickle maintained the `CachedValue` wrapper of the tuple + msgpack does not, so it must be added back in. + """ + try: + value = msgpack.unpackb(value, use_list=False) + return CachedValue(*value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + import fcntl flock_org = fcntl.flock @@ -87,13 +117,16 @@ class CustomLockFactory(FileLock): pass -class FileNamespaceBackend(Serializer, file_backend.DBMBackend): +class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend): + key_prefix = 'file_backend' def __init__(self, arguments): arguments['lock_factory'] = CustomLockFactory super(FileNamespaceBackend, self).__init__(arguments) def list_keys(self, prefix=''): + prefix = '{}:{}'.format(self.key_prefix, prefix) + def cond(v): if not prefix: return True @@ -133,10 +166,9 @@ class FileNamespaceBackend(Serializer, f dbm[key] = self._dumps(value) -class RedisPickleBackend(Serializer, redis_backend.RedisBackend): +class BaseRedisBackend(redis_backend.RedisBackend): def list_keys(self, prefix=''): - if prefix: - prefix = prefix + '*' + prefix = '{}:{}*'.format(self.key_prefix, prefix) return self.client.keys(prefix) def get_store(self): @@ -148,6 +180,15 @@ class RedisPickleBackend(Serializer, red return NO_VALUE return self._loads(value) + def get_multi(self, keys): + if not keys: + return [] + values = self.client.mget(keys) + loads = self._loads + return [ + loads(v) if v is not None else NO_VALUE + for v in values] + def set(self, key, value): if self.redis_expiration_time: self.client.setex(key, self.redis_expiration_time, @@ -156,8 +197,9 @@ class RedisPickleBackend(Serializer, red self.client.set(key, self._dumps(value)) def set_multi(self, mapping): + dumps = self._dumps mapping = dict( - (k, self._dumps(v)) + (k, dumps(v)) for k, v in mapping.items() ) @@ -177,3 +219,13 @@ class RedisPickleBackend(Serializer, red return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep) else: return None + + +class RedisPickleBackend(PickleSerializer, BaseRedisBackend): + key_prefix = 'redis_pickle_backend' + pass + + +class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend): + key_prefix = 'redis_msgpack_backend' + pass diff --git a/vcsserver/lib/rc_cache/utils.py b/vcsserver/lib/rc_cache/utils.py --- a/vcsserver/lib/rc_cache/utils.py +++ b/vcsserver/lib/rc_cache/utils.py @@ -126,13 +126,23 @@ def compute_key_from_params(*args): return sha1("_".join(map(safe_str, args))) -def key_generator(namespace, fn): +def backend_key_generator(backend): + """ + Special wrapper that also sends over the backend to the key generator + """ + def wrapper(namespace, fn): + return key_generator(backend, namespace, fn) + return wrapper + + +def key_generator(backend, namespace, fn): fname = fn.__name__ def generate_key(*args): - namespace_pref = namespace or 'default' + backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix' + namespace_pref = namespace or 'default_namespace' arg_key = compute_key_from_params(*args) - final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) + final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key) return final_key