##// END OF EJS Templates
caches: don't use key_manglers instead prefix keys based on backend.
marcink -
r734:2c6e72c0 default
parent child
Show More
@@ -34,7 +34,7 register_backend(
34 log = logging.getLogger(__name__)
34 log = logging.getLogger(__name__)
35
35
36 from . import region_meta
36 from . import region_meta
37 from .utils import (get_default_cache_settings, key_generator, make_region)
37 from .utils import (get_default_cache_settings, backend_key_generator, make_region)
38
38
39
39
40 def configure_dogpile_cache(settings):
40 def configure_dogpile_cache(settings):
@@ -55,13 +55,12 def configure_dogpile_cache(settings):
55 for region_name in avail_regions:
55 for region_name in avail_regions:
56 new_region = make_region(
56 new_region = make_region(
57 name=region_name,
57 name=region_name,
58 function_key_generator=key_generator
58 function_key_generator=None
59 )
59 )
60
60
61 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
61 new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
62
62 new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
63 log.debug('dogpile: registering a new region %s[%s]',
63 log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
64 region_name, new_region.__dict__)
65 region_meta.dogpile_cache_regions[region_name] = new_region
64 region_meta.dogpile_cache_regions[region_name] = new_region
66
65
67
66
@@ -19,6 +19,8 import time
19 import errno
19 import errno
20 import logging
20 import logging
21
21
22 import msgpack
23 from dogpile.cache.api import CachedValue
22 from dogpile.cache.backends import memory as memory_backend
24 from dogpile.cache.backends import memory as memory_backend
23 from dogpile.cache.backends import file as file_backend
25 from dogpile.cache.backends import file as file_backend
24 from dogpile.cache.backends import redis as redis_backend
26 from dogpile.cache.backends import redis as redis_backend
@@ -34,6 +36,7 log = logging.getLogger(__name__)
34
36
35
37
36 class LRUMemoryBackend(memory_backend.MemoryBackend):
38 class LRUMemoryBackend(memory_backend.MemoryBackend):
39 key_prefix = 'lru_mem_backend'
37 pickle_values = False
40 pickle_values = False
38
41
39 def __init__(self, arguments):
42 def __init__(self, arguments):
@@ -58,7 +61,8 class LRUMemoryBackend(memory_backend.Me
58 self.delete(key)
61 self.delete(key)
59
62
60
63
61 class Serializer(object):
64 class PickleSerializer(object):
65
62 def _dumps(self, value, safe=False):
66 def _dumps(self, value, safe=False):
63 try:
67 try:
64 return compat.pickle.dumps(value)
68 return compat.pickle.dumps(value)
@@ -78,6 +82,32 class Serializer(object):
78 raise
82 raise
79
83
80
84
85 class MsgPackSerializer(object):
86
87 def _dumps(self, value, safe=False):
88 try:
89 return msgpack.packb(value)
90 except Exception:
91 if safe:
92 return NO_VALUE
93 else:
94 raise
95
96 def _loads(self, value, safe=True):
97 """
98 pickle maintained the `CachedValue` wrapper of the tuple
99 msgpack does not, so it must be added back in.
100 """
101 try:
102 value = msgpack.unpackb(value, use_list=False)
103 return CachedValue(*value)
104 except Exception:
105 if safe:
106 return NO_VALUE
107 else:
108 raise
109
110
81 import fcntl
111 import fcntl
82 flock_org = fcntl.flock
112 flock_org = fcntl.flock
83
113
@@ -87,13 +117,16 class CustomLockFactory(FileLock):
87 pass
117 pass
88
118
89
119
90 class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
120 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
121 key_prefix = 'file_backend'
91
122
92 def __init__(self, arguments):
123 def __init__(self, arguments):
93 arguments['lock_factory'] = CustomLockFactory
124 arguments['lock_factory'] = CustomLockFactory
94 super(FileNamespaceBackend, self).__init__(arguments)
125 super(FileNamespaceBackend, self).__init__(arguments)
95
126
96 def list_keys(self, prefix=''):
127 def list_keys(self, prefix=''):
128 prefix = '{}:{}'.format(self.key_prefix, prefix)
129
97 def cond(v):
130 def cond(v):
98 if not prefix:
131 if not prefix:
99 return True
132 return True
@@ -133,10 +166,9 class FileNamespaceBackend(Serializer, f
133 dbm[key] = self._dumps(value)
166 dbm[key] = self._dumps(value)
134
167
135
168
136 class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
169 class BaseRedisBackend(redis_backend.RedisBackend):
137 def list_keys(self, prefix=''):
170 def list_keys(self, prefix=''):
138 if prefix:
171 prefix = '{}:{}*'.format(self.key_prefix, prefix)
139 prefix = prefix + '*'
140 return self.client.keys(prefix)
172 return self.client.keys(prefix)
141
173
142 def get_store(self):
174 def get_store(self):
@@ -148,6 +180,15 class RedisPickleBackend(Serializer, red
148 return NO_VALUE
180 return NO_VALUE
149 return self._loads(value)
181 return self._loads(value)
150
182
183 def get_multi(self, keys):
184 if not keys:
185 return []
186 values = self.client.mget(keys)
187 loads = self._loads
188 return [
189 loads(v) if v is not None else NO_VALUE
190 for v in values]
191
151 def set(self, key, value):
192 def set(self, key, value):
152 if self.redis_expiration_time:
193 if self.redis_expiration_time:
153 self.client.setex(key, self.redis_expiration_time,
194 self.client.setex(key, self.redis_expiration_time,
@@ -156,8 +197,9 class RedisPickleBackend(Serializer, red
156 self.client.set(key, self._dumps(value))
197 self.client.set(key, self._dumps(value))
157
198
158 def set_multi(self, mapping):
199 def set_multi(self, mapping):
200 dumps = self._dumps
159 mapping = dict(
201 mapping = dict(
160 (k, self._dumps(v))
202 (k, dumps(v))
161 for k, v in mapping.items()
203 for k, v in mapping.items()
162 )
204 )
163
205
@@ -177,3 +219,13 class RedisPickleBackend(Serializer, red
177 return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
219 return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
178 else:
220 else:
179 return None
221 return None
222
223
224 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
225 key_prefix = 'redis_pickle_backend'
226 pass
227
228
229 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
230 key_prefix = 'redis_msgpack_backend'
231 pass
@@ -126,13 +126,23 def compute_key_from_params(*args):
126 return sha1("_".join(map(safe_str, args)))
126 return sha1("_".join(map(safe_str, args)))
127
127
128
128
129 def key_generator(namespace, fn):
129 def backend_key_generator(backend):
130 """
131 Special wrapper that also sends over the backend to the key generator
132 """
133 def wrapper(namespace, fn):
134 return key_generator(backend, namespace, fn)
135 return wrapper
136
137
138 def key_generator(backend, namespace, fn):
130 fname = fn.__name__
139 fname = fn.__name__
131
140
132 def generate_key(*args):
141 def generate_key(*args):
133 namespace_pref = namespace or 'default'
142 backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
143 namespace_pref = namespace or 'default_namespace'
134 arg_key = compute_key_from_params(*args)
144 arg_key = compute_key_from_params(*args)
135 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
145 final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
136
146
137 return final_key
147 return final_key
138
148
General Comments 0
You need to be logged in to leave comments. Login now