##// END OF EJS Templates
caches: added redis backend as an option
marcink -
r733:96d998ff default
parent child Browse files
Show More
@@ -22,10 +22,19 b' register_backend('
22 "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
22 "dogpile.cache.rc.memory_lru", "vcsserver.lib.rc_cache.backends",
23 "LRUMemoryBackend")
23 "LRUMemoryBackend")
24
24
25 register_backend(
26 "dogpile.cache.rc.file_namespace", "vcsserver.lib.rc_cache.backends",
27 "FileNamespaceBackend")
28
29 register_backend(
30 "dogpile.cache.rc.redis", "vcsserver.lib.rc_cache.backends",
31 "RedisPickleBackend")
32
33
25 log = logging.getLogger(__name__)
34 log = logging.getLogger(__name__)
26
35
27 from . import region_meta
36 from . import region_meta
28 from .util import key_generator, get_default_cache_settings, make_region
37 from .utils import (get_default_cache_settings, key_generator, make_region)
29
38
30
39
31 def configure_dogpile_cache(settings):
40 def configure_dogpile_cache(settings):
@@ -15,9 +15,16 b''
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import time
19 import errno
18 import logging
20 import logging
19
21
20 from dogpile.cache.backends import memory as memory_backend
22 from dogpile.cache.backends import memory as memory_backend
23 from dogpile.cache.backends import file as file_backend
24 from dogpile.cache.backends import redis as redis_backend
25 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
26 from dogpile.cache.util import memoized_property
27
21 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
28 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
22
29
23
30
@@ -49,3 +56,124 b' class LRUMemoryBackend(memory_backend.Me'
49 def delete_multi(self, keys):
56 def delete_multi(self, keys):
50 for key in keys:
57 for key in keys:
51 self.delete(key)
58 self.delete(key)
59
60
61 class Serializer(object):
62 def _dumps(self, value, safe=False):
63 try:
64 return compat.pickle.dumps(value)
65 except Exception:
66 if safe:
67 return NO_VALUE
68 else:
69 raise
70
71 def _loads(self, value, safe=True):
72 try:
73 return compat.pickle.loads(value)
74 except Exception:
75 if safe:
76 return NO_VALUE
77 else:
78 raise
79
80
81 import fcntl
82 flock_org = fcntl.flock
83
84
85 class CustomLockFactory(FileLock):
86
87 pass
88
89
90 class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
91
92 def __init__(self, arguments):
93 arguments['lock_factory'] = CustomLockFactory
94 super(FileNamespaceBackend, self).__init__(arguments)
95
96 def list_keys(self, prefix=''):
97 def cond(v):
98 if not prefix:
99 return True
100
101 if v.startswith(prefix):
102 return True
103 return False
104
105 with self._dbm_file(True) as dbm:
106
107 return filter(cond, dbm.keys())
108
109 def get_store(self):
110 return self.filename
111
112 def get(self, key):
113 with self._dbm_file(False) as dbm:
114 if hasattr(dbm, 'get'):
115 value = dbm.get(key, NO_VALUE)
116 else:
117 # gdbm objects lack a .get method
118 try:
119 value = dbm[key]
120 except KeyError:
121 value = NO_VALUE
122 if value is not NO_VALUE:
123 value = self._loads(value)
124 return value
125
126 def set(self, key, value):
127 with self._dbm_file(True) as dbm:
128 dbm[key] = self._dumps(value)
129
130 def set_multi(self, mapping):
131 with self._dbm_file(True) as dbm:
132 for key, value in mapping.items():
133 dbm[key] = self._dumps(value)
134
135
136 class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
137 def list_keys(self, prefix=''):
138 if prefix:
139 prefix = prefix + '*'
140 return self.client.keys(prefix)
141
142 def get_store(self):
143 return self.client.connection_pool
144
145 def get(self, key):
146 value = self.client.get(key)
147 if value is None:
148 return NO_VALUE
149 return self._loads(value)
150
151 def set(self, key, value):
152 if self.redis_expiration_time:
153 self.client.setex(key, self.redis_expiration_time,
154 self._dumps(value))
155 else:
156 self.client.set(key, self._dumps(value))
157
158 def set_multi(self, mapping):
159 mapping = dict(
160 (k, self._dumps(v))
161 for k, v in mapping.items()
162 )
163
164 if not self.redis_expiration_time:
165 self.client.mset(mapping)
166 else:
167 pipe = self.client.pipeline()
168 for key, value in mapping.items():
169 pipe.setex(key, self.redis_expiration_time, value)
170 pipe.execute()
171
172 def get_mutex(self, key):
173 u = redis_backend.u
174 if self.distributed_lock:
175 lock_key = u('_lock_{0}').format(key)
176 log.debug('Trying to acquire Redis lock for key %s', lock_key)
177 return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
178 else:
179 return None
@@ -19,10 +19,12 b' import os'
19 import logging
19 import logging
20 import functools
20 import functools
21
21
22 from vcsserver.utils import safe_str, sha1
23 from dogpile.cache import CacheRegion
22 from dogpile.cache import CacheRegion
24 from dogpile.cache.util import compat
23 from dogpile.cache.util import compat
25
24
25 from vcsserver.utils import safe_str, sha1
26
27
26 log = logging.getLogger(__name__)
28 log = logging.getLogger(__name__)
27
29
28
30
@@ -92,6 +94,7 b' class RhodeCodeCacheRegion(CacheRegion):'
92 decorate.get = get
94 decorate.get = get
93 decorate.original = fn
95 decorate.original = fn
94 decorate.key_generator = key_generator
96 decorate.key_generator = key_generator
97 decorate.__wrapped__ = fn
95
98
96 return decorate
99 return decorate
97
100
@@ -110,7 +113,7 b' def get_default_cache_settings(settings,'
110 if key.startswith(prefix):
113 if key.startswith(prefix):
111 name = key.split(prefix)[1].strip()
114 name = key.split(prefix)[1].strip()
112 val = settings[key]
115 val = settings[key]
113 if isinstance(val, basestring):
116 if isinstance(val, compat.string_types):
114 val = val.strip()
117 val = val.strip()
115 cache_settings[name] = val
118 cache_settings[name] = val
116 return cache_settings
119 return cache_settings
General Comments 0
You need to be logged in to leave comments. Login now