Show More
@@ -20,21 +20,22 b'' | |||||
20 |
|
20 | |||
21 | import logging |
|
21 | import logging | |
22 | from dogpile.cache import register_backend |
|
22 | from dogpile.cache import register_backend | |
|
23 | module_name = 'rhodecode' | |||
23 |
|
24 | |||
24 | register_backend( |
|
25 | register_backend( | |
25 |
"dogpile.cache.rc.memory_lru", " |
|
26 | "dogpile.cache.rc.memory_lru", f"{module_name}.lib.rc_cache.backends", | |
26 | "LRUMemoryBackend") |
|
27 | "LRUMemoryBackend") | |
27 |
|
28 | |||
28 | register_backend( |
|
29 | register_backend( | |
29 |
"dogpile.cache.rc.file_namespace", " |
|
30 | "dogpile.cache.rc.file_namespace", f"{module_name}.lib.rc_cache.backends", | |
30 | "FileNamespaceBackend") |
|
31 | "FileNamespaceBackend") | |
31 |
|
32 | |||
32 | register_backend( |
|
33 | register_backend( | |
33 |
"dogpile.cache.rc.redis", " |
|
34 | "dogpile.cache.rc.redis", f"{module_name}.lib.rc_cache.backends", | |
34 | "RedisPickleBackend") |
|
35 | "RedisPickleBackend") | |
35 |
|
36 | |||
36 | register_backend( |
|
37 | register_backend( | |
37 |
"dogpile.cache.rc.redis_msgpack", " |
|
38 | "dogpile.cache.rc.redis_msgpack", f"{module_name}.lib.rc_cache.backends", | |
38 | "RedisMsgPackBackend") |
|
39 | "RedisMsgPackBackend") | |
39 |
|
40 | |||
40 |
|
41 |
@@ -21,22 +21,27 b'' | |||||
21 | import time |
|
21 | import time | |
22 | import errno |
|
22 | import errno | |
23 | import logging |
|
23 | import logging | |
|
24 | import functools | |||
24 |
|
25 | |||
25 | import msgpack |
|
26 | import msgpack | |
26 | import redis |
|
27 | import redis | |
27 | import gevent |
|
28 | import gevent | |
|
29 | import pickle | |||
|
30 | import fcntl | |||
|
31 | flock_org = fcntl.flock | |||
|
32 | from typing import Union | |||
28 |
|
33 | |||
29 | from dogpile.cache.api import CachedValue |
|
|||
30 | from dogpile.cache.backends import memory as memory_backend |
|
34 | from dogpile.cache.backends import memory as memory_backend | |
31 | from dogpile.cache.backends import file as file_backend |
|
35 | from dogpile.cache.backends import file as file_backend | |
32 | from dogpile.cache.backends import redis as redis_backend |
|
36 | from dogpile.cache.backends import redis as redis_backend | |
33 |
from dogpile.cache.backends.file import |
|
37 | from dogpile.cache.backends.file import FileLock | |
34 | from dogpile.cache.util import memoized_property |
|
38 | from dogpile.cache.util import memoized_property | |
|
39 | from dogpile.cache.api import Serializer, Deserializer | |||
35 |
|
40 | |||
36 | from pyramid.settings import asbool |
|
41 | from pyramid.settings import asbool | |
37 |
|
42 | |||
38 | from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug |
|
43 | from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug | |
39 | from rhodecode.lib.utils import safe_str |
|
44 | from rhodecode.lib.str_utils import safe_str | |
40 |
|
45 | |||
41 |
|
46 | |||
42 | _default_max_size = 1024 |
|
47 | _default_max_size = 1024 | |
@@ -70,55 +75,22 b' class LRUMemoryBackend(memory_backend.Me' | |||||
70 | self.delete(key) |
|
75 | self.delete(key) | |
71 |
|
76 | |||
72 |
|
77 | |||
73 |
class PickleSerializer |
|
78 | class PickleSerializer: | |
74 |
|
79 | serializer: Union[None, Serializer] = staticmethod( # type: ignore | ||
75 | def _dumps(self, value, safe=False): |
|
80 | functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL) | |
76 | try: |
|
81 | ) | |
77 | return pickle.dumps(value) |
|
82 | deserializer: Union[None, Deserializer] = staticmethod( # type: ignore | |
78 | except Exception: |
|
83 | functools.partial(pickle.loads) | |
79 | if safe: |
|
84 | ) | |
80 | return NO_VALUE |
|
|||
81 | else: |
|
|||
82 | raise |
|
|||
83 |
|
||||
84 | def _loads(self, value, safe=True): |
|
|||
85 | try: |
|
|||
86 | return pickle.loads(value) |
|
|||
87 | except Exception: |
|
|||
88 | if safe: |
|
|||
89 | return NO_VALUE |
|
|||
90 | else: |
|
|||
91 | raise |
|
|||
92 |
|
85 | |||
93 |
|
86 | |||
94 | class MsgPackSerializer(object): |
|
87 | class MsgPackSerializer(object): | |
95 |
|
88 | serializer: Union[None, Serializer] = staticmethod( # type: ignore | ||
96 | def _dumps(self, value, safe=False): |
|
89 | msgpack.packb | |
97 | try: |
|
90 | ) | |
98 | return msgpack.packb(value) |
|
91 | deserializer: Union[None, Deserializer] = staticmethod( # type: ignore | |
99 | except Exception: |
|
92 | functools.partial(msgpack.unpackb, use_list=False) | |
100 | if safe: |
|
93 | ) | |
101 | return NO_VALUE |
|
|||
102 | else: |
|
|||
103 | raise |
|
|||
104 |
|
||||
105 | def _loads(self, value, safe=True): |
|
|||
106 | """ |
|
|||
107 | pickle maintained the `CachedValue` wrapper of the tuple |
|
|||
108 | msgpack does not, so it must be added back in. |
|
|||
109 | """ |
|
|||
110 | try: |
|
|||
111 | value = msgpack.unpackb(value, use_list=False) |
|
|||
112 | return CachedValue(*value) |
|
|||
113 | except Exception: |
|
|||
114 | if safe: |
|
|||
115 | return NO_VALUE |
|
|||
116 | else: |
|
|||
117 | raise |
|
|||
118 |
|
||||
119 |
|
||||
120 | import fcntl |
|
|||
121 | flock_org = fcntl.flock |
|
|||
122 |
|
94 | |||
123 |
|
95 | |||
124 | class CustomLockFactory(FileLock): |
|
96 | class CustomLockFactory(FileLock): | |
@@ -195,36 +167,6 b' class FileNamespaceBackend(PickleSeriali' | |||||
195 | def get_store(self): |
|
167 | def get_store(self): | |
196 | return self.filename |
|
168 | return self.filename | |
197 |
|
169 | |||
198 | def _dbm_get(self, key): |
|
|||
199 | with self._dbm_file(False) as dbm: |
|
|||
200 | if hasattr(dbm, 'get'): |
|
|||
201 | value = dbm.get(key, NO_VALUE) |
|
|||
202 | else: |
|
|||
203 | # gdbm objects lack a .get method |
|
|||
204 | try: |
|
|||
205 | value = dbm[key] |
|
|||
206 | except KeyError: |
|
|||
207 | value = NO_VALUE |
|
|||
208 | if value is not NO_VALUE: |
|
|||
209 | value = self._loads(value) |
|
|||
210 | return value |
|
|||
211 |
|
||||
212 | def get(self, key): |
|
|||
213 | try: |
|
|||
214 | return self._dbm_get(key) |
|
|||
215 | except Exception: |
|
|||
216 | log.error('Failed to fetch DBM key %s from DB: %s', key, self.get_store()) |
|
|||
217 | raise |
|
|||
218 |
|
||||
219 | def set(self, key, value): |
|
|||
220 | with self._dbm_file(True) as dbm: |
|
|||
221 | dbm[key] = self._dumps(value) |
|
|||
222 |
|
||||
223 | def set_multi(self, mapping): |
|
|||
224 | with self._dbm_file(True) as dbm: |
|
|||
225 | for key, value in mapping.items(): |
|
|||
226 | dbm[key] = self._dumps(value) |
|
|||
227 |
|
||||
228 |
|
170 | |||
229 | class BaseRedisBackend(redis_backend.RedisBackend): |
|
171 | class BaseRedisBackend(redis_backend.RedisBackend): | |
230 | key_prefix = '' |
|
172 | key_prefix = '' | |
@@ -251,58 +193,26 b' class BaseRedisBackend(redis_backend.Red' | |||||
251 | ) |
|
193 | ) | |
252 |
|
194 | |||
253 | connection_pool = redis.ConnectionPool(**args) |
|
195 | connection_pool = redis.ConnectionPool(**args) | |
254 |
|
196 | self.writer_client = redis.StrictRedis( | ||
255 |
|
|
197 | connection_pool=connection_pool | |
|
198 | ) | |||
|
199 | self.reader_client = self.writer_client | |||
256 |
|
200 | |||
257 | def list_keys(self, prefix=''): |
|
201 | def list_keys(self, prefix=''): | |
258 | prefix = '{}:{}*'.format(self.key_prefix, prefix) |
|
202 | prefix = '{}:{}*'.format(self.key_prefix, prefix) | |
259 | return self.client.keys(prefix) |
|
203 | return self.reader_client.keys(prefix) | |
260 |
|
204 | |||
261 | def get_store(self): |
|
205 | def get_store(self): | |
262 | return self.client.connection_pool |
|
206 | return self.reader_client.connection_pool | |
263 |
|
||||
264 | def get(self, key): |
|
|||
265 | value = self.client.get(key) |
|
|||
266 | if value is None: |
|
|||
267 | return NO_VALUE |
|
|||
268 | return self._loads(value) |
|
|||
269 |
|
||||
270 | def get_multi(self, keys): |
|
|||
271 | if not keys: |
|
|||
272 | return [] |
|
|||
273 | values = self.client.mget(keys) |
|
|||
274 | loads = self._loads |
|
|||
275 | return [ |
|
|||
276 | loads(v) if v is not None else NO_VALUE |
|
|||
277 | for v in values] |
|
|||
278 |
|
||||
279 | def set(self, key, value): |
|
|||
280 | if self.redis_expiration_time: |
|
|||
281 | self.client.setex(key, self.redis_expiration_time, |
|
|||
282 | self._dumps(value)) |
|
|||
283 | else: |
|
|||
284 | self.client.set(key, self._dumps(value)) |
|
|||
285 |
|
||||
286 | def set_multi(self, mapping): |
|
|||
287 | dumps = self._dumps |
|
|||
288 | mapping = dict( |
|
|||
289 | (k, dumps(v)) |
|
|||
290 | for k, v in mapping.items() |
|
|||
291 | ) |
|
|||
292 |
|
||||
293 | if not self.redis_expiration_time: |
|
|||
294 | self.client.mset(mapping) |
|
|||
295 | else: |
|
|||
296 | pipe = self.client.pipeline() |
|
|||
297 | for key, value in mapping.items(): |
|
|||
298 | pipe.setex(key, self.redis_expiration_time, value) |
|
|||
299 | pipe.execute() |
|
|||
300 |
|
207 | |||
301 | def get_mutex(self, key): |
|
208 | def get_mutex(self, key): | |
302 | if self.distributed_lock: |
|
209 | if self.distributed_lock: | |
303 | lock_key = '_lock_{0}'.format(safe_str(key)) |
|
210 | lock_key = '_lock_{0}'.format(safe_str(key)) | |
304 |
return get_mutex_lock( |
|
211 | return get_mutex_lock( | |
305 | auto_renewal=self._lock_auto_renewal) |
|
212 | self.writer_client, lock_key, | |
|
213 | self._lock_timeout, | |||
|
214 | auto_renewal=self._lock_auto_renewal | |||
|
215 | ) | |||
306 | else: |
|
216 | else: | |
307 | return None |
|
217 | return None | |
308 |
|
218 | |||
@@ -318,7 +228,7 b' class RedisMsgPackBackend(MsgPackSeriali' | |||||
318 |
|
228 | |||
319 |
|
229 | |||
320 | def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False): |
|
230 | def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False): | |
321 | import redis_lock |
|
231 | from rhodecode.lib._vendor import redis_lock | |
322 |
|
232 | |||
323 | class _RedisLockWrapper(object): |
|
233 | class _RedisLockWrapper(object): | |
324 | """LockWrapper for redis_lock""" |
|
234 | """LockWrapper for redis_lock""" |
General Comments 0
You need to be logged in to leave comments.
Login now