diff --git a/rhodecode/lib/rc_cache/__init__.py b/rhodecode/lib/rc_cache/__init__.py --- a/rhodecode/lib/rc_cache/__init__.py +++ b/rhodecode/lib/rc_cache/__init__.py @@ -33,12 +33,16 @@ register_backend( "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends", "RedisPickleBackend") +register_backend( + "dogpile.cache.rc.redis_msgpack", "rhodecode.lib.rc_cache.backends", + "RedisMsgPackBackend") + log = logging.getLogger(__name__) from . import region_meta from .utils import ( - get_default_cache_settings, key_generator, get_or_create_region, + get_default_cache_settings, backend_key_generator, get_or_create_region, clear_cache_namespace, make_region, InvalidationContext, FreshRegionCache, ActiveRegionCache) @@ -61,13 +65,12 @@ def configure_dogpile_cache(settings): for region_name in avail_regions: new_region = make_region( name=region_name, - function_key_generator=key_generator + function_key_generator=None ) new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name)) - - log.debug('dogpile: registering a new region %s[%s]', - region_name, new_region.__dict__) + new_region.function_key_generator = backend_key_generator(new_region.actual_backend) + log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__) region_meta.dogpile_cache_regions[region_name] = new_region diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -22,8 +22,10 @@ import time import errno import logging +import msgpack import gevent +from dogpile.cache.api import CachedValue from dogpile.cache.backends import memory as memory_backend from dogpile.cache.backends import file as file_backend from dogpile.cache.backends import redis as redis_backend @@ -39,6 +41,7 @@ log = logging.getLogger(__name__) class LRUMemoryBackend(memory_backend.MemoryBackend): + key_prefix = 'lru_mem_backend' pickle_values = False def __init__(self, arguments): @@ -63,7 +66,8 @@ class LRUMemoryBackend(memory_backend.Me self.delete(key) -class Serializer(object): +class PickleSerializer(object): + def _dumps(self, value, safe=False): try: return compat.pickle.dumps(value) @@ -83,6 +87,32 @@ class Serializer(object): raise +class MsgPackSerializer(object): + + def _dumps(self, value, safe=False): + try: + return msgpack.packb(value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + def _loads(self, value, safe=True): + """ + pickle maintained the `CachedValue` wrapper of the tuple + msgpack does not, so it must be added back in. + """ + try: + value = msgpack.unpackb(value, use_list=False) + return CachedValue(*value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + import fcntl flock_org = fcntl.flock @@ -123,13 +153,16 @@ class CustomLockFactory(FileLock): return fcntl -class FileNamespaceBackend(Serializer, file_backend.DBMBackend): +class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend): + key_prefix = 'file_backend' def __init__(self, arguments): arguments['lock_factory'] = CustomLockFactory super(FileNamespaceBackend, self).__init__(arguments) def list_keys(self, prefix=''): + prefix = '{}:{}'.format(self.key_prefix, prefix) + def cond(v): if not prefix: return True @@ -169,10 +202,9 @@ class FileNamespaceBackend(Serializer, f dbm[key] = self._dumps(value) -class RedisPickleBackend(Serializer, redis_backend.RedisBackend): +class BaseRedisBackend(redis_backend.RedisBackend): def list_keys(self, prefix=''): - if prefix: - prefix = prefix + '*' + prefix = '{}:{}*'.format(self.key_prefix, prefix) return self.client.keys(prefix) def get_store(self): @@ -184,6 +216,15 @@ class RedisPickleBackend(Serializer, red return NO_VALUE return self._loads(value) + def get_multi(self, keys): + if not keys: + return [] + values = self.client.mget(keys) + loads = self._loads + return [ + loads(v) if v is not None else NO_VALUE + for v in values] + def set(self, key, value): if self.redis_expiration_time: self.client.setex(key, self.redis_expiration_time, @@ -192,8 +233,9 @@ class RedisPickleBackend(Serializer, red self.client.set(key, self._dumps(value)) def set_multi(self, mapping): + dumps = self._dumps mapping = dict( - (k, self._dumps(v)) + (k, dumps(v)) for k, v in mapping.items() ) @@ -213,3 +255,13 @@ class RedisPickleBackend(Serializer, red return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep) else: return None + + +class RedisPickleBackend(PickleSerializer, BaseRedisBackend): + key_prefix = 'redis_pickle_backend' + pass + + +class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend): + key_prefix = 'redis_msgpack_backend' + pass diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -134,13 +134,23 @@ def compute_key_from_params(*args): return sha1("_".join(map(safe_str, args))) -def key_generator(namespace, fn): +def backend_key_generator(backend): + """ + Special wrapper that also sends over the backend to the key generator + """ + def wrapper(namespace, fn): + return key_generator(backend, namespace, fn) + return wrapper + + +def key_generator(backend, namespace, fn): fname = fn.__name__ def generate_key(*args): - namespace_pref = namespace or 'default' + backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix' + namespace_pref = namespace or 'default_namespace' arg_key = compute_key_from_params(*args) - final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key) + final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key) return final_key @@ -167,7 +177,8 @@ def get_or_create_region(region_name, re if not os.path.isdir(cache_dir): os.makedirs(cache_dir) new_region = make_region( - name=region_uid_name, function_key_generator=key_generator + name=region_uid_name, + function_key_generator=backend_key_generator(region_obj.actual_backend) ) namespace_filename = os.path.join( cache_dir, "{}.cache.dbm".format(region_namespace)) @@ -179,7 +190,7 @@ def get_or_create_region(region_name, re ) # create and save in region caches - log.debug('configuring new region: %s',region_uid_name) + log.debug('configuring new region: %s', region_uid_name) region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region return region_obj diff --git a/rhodecode/tests/lib/test_caches.py b/rhodecode/tests/lib/test_caches.py --- a/rhodecode/tests/lib/test_caches.py +++ b/rhodecode/tests/lib/test_caches.py @@ -67,7 +67,7 @@ class TestCaches(object): def test_cache_keygen(self, example_input, example_namespace): def func_wrapped(): return 1 - func = rc_cache.utils.key_generator(example_namespace, func_wrapped) + func = rc_cache.utils.key_generator(None, example_namespace, func_wrapped) key = func(*example_input) assert key