caches: added redis backend as an option

vcsserver/lib/rc_cache/__init__.py

@@ -22,10 +22,19 b' register_backend('
     "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
+register_backend(
+    "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
+    "FileNamespaceBackend")
+
+register_backend(
+    "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
+    "RedisPickleBackend")
+
+
 log = logging.getLogger(__name__)
 
 from . import region_meta
-from .util import key_generator, get_default_cache_settings, make_region
+from .utils import (get_default_cache_settings, key_generator, make_region)
 
 
 def configure_dogpile_cache(settings):

vcsserver/lib/rc_cache/backends.py

@@ -15,9 +15,16 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import time
+import errno
 import logging
 
 from dogpile.cache.backends import memory as memory_backend
+from dogpile.cache.backends import file as file_backend
+from dogpile.cache.backends import redis as redis_backend
+from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
+from dogpile.cache.util import memoized_property
+
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
@@ -49,3 +56,124 b' class LRUMemoryBackend(memory_backend.Me'
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
+
+
+class Serializer(object):
+    def _dumps(self, value, safe=False):
+        try:
+            return compat.pickle.dumps(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        try:
+            return compat.pickle.loads(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
+import fcntl
+flock_org = fcntl.flock
+
+
+class CustomLockFactory(FileLock):
+
+    pass
+
+
+class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+
+    def __init__(self, arguments):
+        arguments['lock_factory'] = CustomLockFactory
+        super(FileNamespaceBackend, self).__init__(arguments)
+
+    def list_keys(self, prefix=''):
+        def cond(v):
+            if not prefix:
+                return True
+
+            if v.startswith(prefix):
+                return True
+            return False
+
+        with self._dbm_file(True) as dbm:
+
+            return filter(cond, dbm.keys())
+
+    def get_store(self):
+        return self.filename
+
+    def get(self, key):
+        with self._dbm_file(False) as dbm:
+            if hasattr(dbm, 'get'):
+                value = dbm.get(key, NO_VALUE)
+            else:
+                # gdbm objects lack a .get method
+                try:
+                    value = dbm[key]
+                except KeyError:
+                    value = NO_VALUE
+            if value is not NO_VALUE:
+                value = self._loads(value)
+            return value
+
+    def set(self, key, value):
+        with self._dbm_file(True) as dbm:
+            dbm[key] = self._dumps(value)
+
+    def set_multi(self, mapping):
+        with self._dbm_file(True) as dbm:
+            for key, value in mapping.items():
+                dbm[key] = self._dumps(value)
+
+
+class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+    def list_keys(self, prefix=''):
+        if prefix:
+            prefix = prefix + '*'
+        return self.client.keys(prefix)
+
+    def get_store(self):
+        return self.client.connection_pool
+
+    def get(self, key):
+        value = self.client.get(key)
+        if value is None:
+            return NO_VALUE
+        return self._loads(value)
+
+    def set(self, key, value):
+        if self.redis_expiration_time:
+            self.client.setex(key, self.redis_expiration_time,
+                              self._dumps(value))
+        else:
+            self.client.set(key, self._dumps(value))
+
+    def set_multi(self, mapping):
+        mapping = dict(
+            (k, self._dumps(v))
+            for k, v in mapping.items()
+        )
+
+        if not self.redis_expiration_time:
+            self.client.mset(mapping)
+        else:
+            pipe = self.client.pipeline()
+            for key, value in mapping.items():
+                pipe.setex(key, self.redis_expiration_time, value)
+            pipe.execute()
+
+    def get_mutex(self, key):
+        u = redis_backend.u
+        if self.distributed_lock:
+            lock_key = u('_lock_{0}').format(key)
+            log.debug('Trying to acquire Redis lock for key %s', lock_key)
+            return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
+        else:
+            return None

vcsserver/lib/rc_cache/utils.py

@@ -19,10 +19,12 b' import os'
 import logging
 import functools
 
-from vcsserver.utils import safe_str, sha1
 from dogpile.cache import CacheRegion
 from dogpile.cache.util import compat
 
+from vcsserver.utils import safe_str, sha1
+
+
 log = logging.getLogger(__name__)
 
 
@@ -92,6 +94,7 b' class RhodeCodeCacheRegion(CacheRegion):'
             decorate.get = get
             decorate.original = fn
             decorate.key_generator = key_generator
+            decorate.__wrapped__ = fn
 
             return decorate
 
@@ -110,7 +113,7 b' def get_default_cache_settings(settings,'
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
-                if isinstance(val, basestring):
+                if isinstance(val, compat.string_types):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings