diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py --- a/vcsserver/lib/rc_cache/__init__.py +++ b/vcsserver/lib/rc_cache/__init__.py @@ -22,10 +22,19 @@ register_backend( "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends", "LRUMemoryBackend") +register_backend( + "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends", + "FileNamespaceBackend") + +register_backend( + "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends", + "RedisPickleBackend") + + log = logging.getLogger(__name__) from . import region_meta -from .util import key_generator, get_default_cache_settings, make_region +from .utils import (get_default_cache_settings, key_generator, make_region) def configure_dogpile_cache(settings): diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py --- a/vcsserver/lib/rc_cache/backends.py +++ b/vcsserver/lib/rc_cache/backends.py @@ -15,9 +15,16 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import time +import errno import logging from dogpile.cache.backends import memory as memory_backend +from dogpile.cache.backends import file as file_backend +from dogpile.cache.backends import redis as redis_backend +from dogpile.cache.backends.file import NO_VALUE, compat, FileLock +from dogpile.cache.util import memoized_property + from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug @@ -49,3 +56,124 @@ class LRUMemoryBackend(memory_backend.Me def delete_multi(self, keys): for key in keys: self.delete(key) + + +class Serializer(object): + def _dumps(self, value, safe=False): + try: + return compat.pickle.dumps(value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + def _loads(self, value, safe=True): + try: + return compat.pickle.loads(value) + except Exception: + if safe: + return NO_VALUE + else: + raise + + +import fcntl +flock_org = fcntl.flock + + +class CustomLockFactory(FileLock): + + pass + + +class FileNamespaceBackend(Serializer, file_backend.DBMBackend): + + def __init__(self, arguments): + arguments['lock_factory'] = CustomLockFactory + super(FileNamespaceBackend, self).__init__(arguments) + + def list_keys(self, prefix=''): + def cond(v): + if not prefix: + return True + + if v.startswith(prefix): + return True + return False + + with self._dbm_file(True) as dbm: + + return filter(cond, dbm.keys()) + + def get_store(self): + return self.filename + + def get(self, key): + with self._dbm_file(False) as dbm: + if hasattr(dbm, 'get'): + value = dbm.get(key, NO_VALUE) + else: + # gdbm objects lack a .get method + try: + value = dbm[key] + except KeyError: + value = NO_VALUE + if value is not NO_VALUE: + value = self._loads(value) + return value + + def set(self, key, value): + with self._dbm_file(True) as dbm: + dbm[key] = self._dumps(value) + + def set_multi(self, mapping): + with self._dbm_file(True) as dbm: + for key, value in mapping.items(): + dbm[key] = self._dumps(value) + + +class RedisPickleBackend(Serializer, redis_backend.RedisBackend): + def list_keys(self, prefix=''): + if prefix: + prefix = prefix + '*' + return self.client.keys(prefix) + + def get_store(self): + return self.client.connection_pool + + def get(self, key): + value = self.client.get(key) + if value is None: + return NO_VALUE + return self._loads(value) + + def set(self, key, value): + if self.redis_expiration_time: + self.client.setex(key, self.redis_expiration_time, + self._dumps(value)) + else: + self.client.set(key, self._dumps(value)) + + def set_multi(self, mapping): + mapping = dict( + (k, self._dumps(v)) + for k, v in mapping.items() + ) + + if not self.redis_expiration_time: + self.client.mset(mapping) + else: + pipe = self.client.pipeline() + for key, value in mapping.items(): + pipe.setex(key, self.redis_expiration_time, value) + pipe.execute() + + def get_mutex(self, key): + u = redis_backend.u + if self.distributed_lock: + lock_key = u('_lock_{0}').format(key) + log.debug('Trying to acquire Redis lock for key %s', lock_key) + return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep) + else: + return None diff --git a/vcsserver/lib/rc_cache/util.py b/vcsserver/lib/rc_cache/utils.py rename from vcsserver/lib/rc_cache/util.py rename to vcsserver/lib/rc_cache/utils.py --- a/vcsserver/lib/rc_cache/util.py +++ b/vcsserver/lib/rc_cache/utils.py @@ -19,10 +19,12 @@ import os import logging import functools -from vcsserver.utils import safe_str, sha1 from dogpile.cache import CacheRegion from dogpile.cache.util import compat +from vcsserver.utils import safe_str, sha1 + + log = logging.getLogger(__name__) @@ -92,6 +94,7 @@ class RhodeCodeCacheRegion(CacheRegion): decorate.get = get decorate.original = fn decorate.key_generator = key_generator + decorate.__wrapped__ = fn return decorate @@ -110,7 +113,7 @@ def get_default_cache_settings(settings, if key.startswith(prefix): name = key.split(prefix)[1].strip() val = settings[key] - if isinstance(val, basestring): + if isinstance(val, compat.string_types): val = val.strip() cache_settings[name] = val return cache_settings