##// END OF EJS Templates
rccache: refactor and update code to support latest dogpile code changes (mostly on custom serializers)
super-admin -
r4985:fe735b46 default
parent child Browse files
Show More
@@ -20,21 +20,22 b''
20
20
21 import logging
21 import logging
22 from dogpile.cache import register_backend
22 from dogpile.cache import register_backend
23 module_name = 'rhodecode'
23
24
24 register_backend(
25 register_backend(
25 "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
26 "dogpile.cache.rc.memory_lru", f"{module_name}.lib.rc_cache.backends",
26 "LRUMemoryBackend")
27 "LRUMemoryBackend")
27
28
28 register_backend(
29 register_backend(
29 "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
30 "dogpile.cache.rc.file_namespace", f"{module_name}.lib.rc_cache.backends",
30 "FileNamespaceBackend")
31 "FileNamespaceBackend")
31
32
32 register_backend(
33 register_backend(
33 "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
34 "dogpile.cache.rc.redis", f"{module_name}.lib.rc_cache.backends",
34 "RedisPickleBackend")
35 "RedisPickleBackend")
35
36
36 register_backend(
37 register_backend(
37 "dogpile.cache.rc.redis_msgpack", "rhodecode.lib.rc_cache.backends",
38 "dogpile.cache.rc.redis_msgpack", f"{module_name}.lib.rc_cache.backends",
38 "RedisMsgPackBackend")
39 "RedisMsgPackBackend")
39
40
40
41
@@ -21,22 +21,27 b''
21 import time
21 import time
22 import errno
22 import errno
23 import logging
23 import logging
24 import functools
24
25
25 import msgpack
26 import msgpack
26 import redis
27 import redis
27 import gevent
28 import gevent
29 import pickle
30 import fcntl
31 flock_org = fcntl.flock
32 from typing import Union
28
33
29 from dogpile.cache.api import CachedValue
30 from dogpile.cache.backends import memory as memory_backend
34 from dogpile.cache.backends import memory as memory_backend
31 from dogpile.cache.backends import file as file_backend
35 from dogpile.cache.backends import file as file_backend
32 from dogpile.cache.backends import redis as redis_backend
36 from dogpile.cache.backends import redis as redis_backend
33 from dogpile.cache.backends.file import NO_VALUE, FileLock
37 from dogpile.cache.backends.file import FileLock
34 from dogpile.cache.util import memoized_property
38 from dogpile.cache.util import memoized_property
39 from dogpile.cache.api import Serializer, Deserializer
35
40
36 from pyramid.settings import asbool
41 from pyramid.settings import asbool
37
42
38 from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
43 from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
39 from rhodecode.lib.utils import safe_str
44 from rhodecode.lib.str_utils import safe_str
40
45
41
46
42 _default_max_size = 1024
47 _default_max_size = 1024
@@ -70,55 +75,22 b' class LRUMemoryBackend(memory_backend.Me'
70 self.delete(key)
75 self.delete(key)
71
76
72
77
73 class PickleSerializer(object):
78 class PickleSerializer:
74
79 serializer: Union[None, Serializer] = staticmethod( # type: ignore
75 def _dumps(self, value, safe=False):
80 functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
76 try:
81 )
77 return pickle.dumps(value)
82 deserializer: Union[None, Deserializer] = staticmethod( # type: ignore
78 except Exception:
83 functools.partial(pickle.loads)
79 if safe:
84 )
80 return NO_VALUE
81 else:
82 raise
83
84 def _loads(self, value, safe=True):
85 try:
86 return pickle.loads(value)
87 except Exception:
88 if safe:
89 return NO_VALUE
90 else:
91 raise
92
85
93
86
94 class MsgPackSerializer(object):
87 class MsgPackSerializer(object):
95
88 serializer: Union[None, Serializer] = staticmethod( # type: ignore
96 def _dumps(self, value, safe=False):
89 msgpack.packb
97 try:
90 )
98 return msgpack.packb(value)
91 deserializer: Union[None, Deserializer] = staticmethod( # type: ignore
99 except Exception:
92 functools.partial(msgpack.unpackb, use_list=False)
100 if safe:
93 )
101 return NO_VALUE
102 else:
103 raise
104
105 def _loads(self, value, safe=True):
106 """
107 pickle maintained the `CachedValue` wrapper of the tuple
108 msgpack does not, so it must be added back in.
109 """
110 try:
111 value = msgpack.unpackb(value, use_list=False)
112 return CachedValue(*value)
113 except Exception:
114 if safe:
115 return NO_VALUE
116 else:
117 raise
118
119
120 import fcntl
121 flock_org = fcntl.flock
122
94
123
95
124 class CustomLockFactory(FileLock):
96 class CustomLockFactory(FileLock):
@@ -195,36 +167,6 b' class FileNamespaceBackend(PickleSeriali'
195 def get_store(self):
167 def get_store(self):
196 return self.filename
168 return self.filename
197
169
198 def _dbm_get(self, key):
199 with self._dbm_file(False) as dbm:
200 if hasattr(dbm, 'get'):
201 value = dbm.get(key, NO_VALUE)
202 else:
203 # gdbm objects lack a .get method
204 try:
205 value = dbm[key]
206 except KeyError:
207 value = NO_VALUE
208 if value is not NO_VALUE:
209 value = self._loads(value)
210 return value
211
212 def get(self, key):
213 try:
214 return self._dbm_get(key)
215 except Exception:
216 log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store())
217 raise
218
219 def set(self, key, value):
220 with self._dbm_file(True) as dbm:
221 dbm[key] = self._dumps(value)
222
223 def set_multi(self, mapping):
224 with self._dbm_file(True) as dbm:
225 for key, value in mapping.items():
226 dbm[key] = self._dumps(value)
227
228
170
229 class BaseRedisBackend(redis_backend.RedisBackend):
171 class BaseRedisBackend(redis_backend.RedisBackend):
230 key_prefix = ''
172 key_prefix = ''
@@ -251,58 +193,26 b' class BaseRedisBackend(redis_backend.Red'
251 )
193 )
252
194
253 connection_pool = redis.ConnectionPool(**args)
195 connection_pool = redis.ConnectionPool(**args)
254
196 self.writer_client = redis.StrictRedis(
255 return redis.StrictRedis(connection_pool=connection_pool)
197 connection_pool=connection_pool
198 )
199 self.reader_client = self.writer_client
256
200
257 def list_keys(self, prefix=''):
201 def list_keys(self, prefix=''):
258 prefix = '{}:{}*'.format(self.key_prefix, prefix)
202 prefix = '{}:{}*'.format(self.key_prefix, prefix)
259 return self.client.keys(prefix)
203 return self.reader_client.keys(prefix)
260
204
261 def get_store(self):
205 def get_store(self):
262 return self.client.connection_pool
206 return self.reader_client.connection_pool
263
264 def get(self, key):
265 value = self.client.get(key)
266 if value is None:
267 return NO_VALUE
268 return self._loads(value)
269
270 def get_multi(self, keys):
271 if not keys:
272 return []
273 values = self.client.mget(keys)
274 loads = self._loads
275 return [
276 loads(v) if v is not None else NO_VALUE
277 for v in values]
278
279 def set(self, key, value):
280 if self.redis_expiration_time:
281 self.client.setex(key, self.redis_expiration_time,
282 self._dumps(value))
283 else:
284 self.client.set(key, self._dumps(value))
285
286 def set_multi(self, mapping):
287 dumps = self._dumps
288 mapping = dict(
289 (k, dumps(v))
290 for k, v in mapping.items()
291 )
292
293 if not self.redis_expiration_time:
294 self.client.mset(mapping)
295 else:
296 pipe = self.client.pipeline()
297 for key, value in mapping.items():
298 pipe.setex(key, self.redis_expiration_time, value)
299 pipe.execute()
300
207
301 def get_mutex(self, key):
208 def get_mutex(self, key):
302 if self.distributed_lock:
209 if self.distributed_lock:
303 lock_key = '_lock_{0}'.format(safe_str(key))
210 lock_key = '_lock_{0}'.format(safe_str(key))
304 return get_mutex_lock(self.client, lock_key, self._lock_timeout,
211 return get_mutex_lock(
305 auto_renewal=self._lock_auto_renewal)
212 self.writer_client, lock_key,
213 self._lock_timeout,
214 auto_renewal=self._lock_auto_renewal
215 )
306 else:
216 else:
307 return None
217 return None
308
218
@@ -318,7 +228,7 b' class RedisMsgPackBackend(MsgPackSeriali'
318
228
319
229
320 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
230 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
321 import redis_lock
231 from rhodecode.lib._vendor import redis_lock
322
232
323 class _RedisLockWrapper(object):
233 class _RedisLockWrapper(object):
324 """LockWrapper for redis_lock"""
234 """LockWrapper for redis_lock"""
General Comments 0
You need to be logged in to leave comments. Login now