##// END OF EJS Templates
fixed error message about cache settings
marcink -
r610:b0a411f5 default
parent child Browse files
Show More
@@ -1,275 +1,276
1 """caching_query.py
1 """caching_query.py
2
2
3 Represent persistence structures which allow the usage of
3 Represent persistence structures which allow the usage of
4 Beaker caching with SQLAlchemy.
4 Beaker caching with SQLAlchemy.
5
5
6 The three new concepts introduced here are:
6 The three new concepts introduced here are:
7
7
8 * CachingQuery - a Query subclass that caches and
8 * CachingQuery - a Query subclass that caches and
9 retrieves results in/from Beaker.
9 retrieves results in/from Beaker.
10 * FromCache - a query option that establishes caching
10 * FromCache - a query option that establishes caching
11 parameters on a Query
11 parameters on a Query
12 * RelationshipCache - a variant of FromCache which is specific
12 * RelationshipCache - a variant of FromCache which is specific
13 to a query invoked during a lazy load.
13 to a query invoked during a lazy load.
14 * _params_from_query - extracts value parameters from
14 * _params_from_query - extracts value parameters from
15 a Query.
15 a Query.
16
16
17 The rest of what's here are standard SQLAlchemy and
17 The rest of what's here are standard SQLAlchemy and
18 Beaker constructs.
18 Beaker constructs.
19
19
20 """
20 """
21 from beaker.exceptions import BeakerException
21 from beaker.exceptions import BeakerException
22 from sqlalchemy.orm.interfaces import MapperOption
22 from sqlalchemy.orm.interfaces import MapperOption
23 from sqlalchemy.orm.query import Query
23 from sqlalchemy.orm.query import Query
24 from sqlalchemy.sql import visitors
24 from sqlalchemy.sql import visitors
25 import beaker
25 import beaker
26
26
27 class CachingQuery(Query):
27 class CachingQuery(Query):
28 """A Query subclass which optionally loads full results from a Beaker
28 """A Query subclass which optionally loads full results from a Beaker
29 cache region.
29 cache region.
30
30
31 The CachingQuery stores additional state that allows it to consult
31 The CachingQuery stores additional state that allows it to consult
32 a Beaker cache before accessing the database:
32 a Beaker cache before accessing the database:
33
33
34 * A "region", which is a cache region argument passed to a
34 * A "region", which is a cache region argument passed to a
35 Beaker CacheManager, specifies a particular cache configuration
35 Beaker CacheManager, specifies a particular cache configuration
36 (including backend implementation, expiration times, etc.)
36 (including backend implementation, expiration times, etc.)
37 * A "namespace", which is a qualifying name that identifies a
37 * A "namespace", which is a qualifying name that identifies a
38 group of keys within the cache. A query that filters on a name
38 group of keys within the cache. A query that filters on a name
39 might use the name "by_name", a query that filters on a date range
39 might use the name "by_name", a query that filters on a date range
40 to a joined table might use the name "related_date_range".
40 to a joined table might use the name "related_date_range".
41
41
42 When the above state is present, a Beaker cache is retrieved.
42 When the above state is present, a Beaker cache is retrieved.
43
43
44 The "namespace" name is first concatenated with
44 The "namespace" name is first concatenated with
45 a string composed of the individual entities and columns the Query
45 a string composed of the individual entities and columns the Query
46 requests, i.e. such as ``Query(User.id, User.name)``.
46 requests, i.e. such as ``Query(User.id, User.name)``.
47
47
48 The Beaker cache is then loaded from the cache manager based
48 The Beaker cache is then loaded from the cache manager based
49 on the region and composed namespace. The key within the cache
49 on the region and composed namespace. The key within the cache
50 itself is then constructed against the bind parameters specified
50 itself is then constructed against the bind parameters specified
51 by this query, which are usually literals defined in the
51 by this query, which are usually literals defined in the
52 WHERE clause.
52 WHERE clause.
53
53
54 The FromCache and RelationshipCache mapper options below represent
54 The FromCache and RelationshipCache mapper options below represent
55 the "public" method of configuring this state upon the CachingQuery.
55 the "public" method of configuring this state upon the CachingQuery.
56
56
57 """
57 """
58
58
59 def __init__(self, manager, *args, **kw):
59 def __init__(self, manager, *args, **kw):
60 self.cache_manager = manager
60 self.cache_manager = manager
61 Query.__init__(self, *args, **kw)
61 Query.__init__(self, *args, **kw)
62
62
63 def __iter__(self):
63 def __iter__(self):
64 """override __iter__ to pull results from Beaker
64 """override __iter__ to pull results from Beaker
65 if particular attributes have been configured.
65 if particular attributes have been configured.
66
66
67 Note that this approach does *not* detach the loaded objects from
67 Note that this approach does *not* detach the loaded objects from
68 the current session. If the cache backend is an in-process cache
68 the current session. If the cache backend is an in-process cache
69 (like "memory") and lives beyond the scope of the current session's
69 (like "memory") and lives beyond the scope of the current session's
70 transaction, those objects may be expired. The method here can be
70 transaction, those objects may be expired. The method here can be
71 modified to first expunge() each loaded item from the current
71 modified to first expunge() each loaded item from the current
72 session before returning the list of items, so that the items
72 session before returning the list of items, so that the items
73 in the cache are not the same ones in the current Session.
73 in the cache are not the same ones in the current Session.
74
74
75 """
75 """
76 if hasattr(self, '_cache_parameters'):
76 if hasattr(self, '_cache_parameters'):
77 return self.get_value(createfunc=lambda: list(Query.__iter__(self)))
77 return self.get_value(createfunc=lambda: list(Query.__iter__(self)))
78 else:
78 else:
79 return Query.__iter__(self)
79 return Query.__iter__(self)
80
80
81 def invalidate(self):
81 def invalidate(self):
82 """Invalidate the value represented by this Query."""
82 """Invalidate the value represented by this Query."""
83
83
84 cache, cache_key = _get_cache_parameters(self)
84 cache, cache_key = _get_cache_parameters(self)
85 cache.remove(cache_key)
85 cache.remove(cache_key)
86
86
87 def get_value(self, merge=True, createfunc=None):
87 def get_value(self, merge=True, createfunc=None):
88 """Return the value from the cache for this query.
88 """Return the value from the cache for this query.
89
89
90 Raise KeyError if no value present and no
90 Raise KeyError if no value present and no
91 createfunc specified.
91 createfunc specified.
92
92
93 """
93 """
94 cache, cache_key = _get_cache_parameters(self)
94 cache, cache_key = _get_cache_parameters(self)
95 ret = cache.get_value(cache_key, createfunc=createfunc)
95 ret = cache.get_value(cache_key, createfunc=createfunc)
96 if merge:
96 if merge:
97 ret = self.merge_result(ret, load=False)
97 ret = self.merge_result(ret, load=False)
98 return ret
98 return ret
99
99
100 def set_value(self, value):
100 def set_value(self, value):
101 """Set the value in the cache for this query."""
101 """Set the value in the cache for this query."""
102
102
103 cache, cache_key = _get_cache_parameters(self)
103 cache, cache_key = _get_cache_parameters(self)
104 cache.put(cache_key, value)
104 cache.put(cache_key, value)
105
105
106 def query_callable(manager):
106 def query_callable(manager):
107 def query(*arg, **kw):
107 def query(*arg, **kw):
108 return CachingQuery(manager, *arg, **kw)
108 return CachingQuery(manager, *arg, **kw)
109 return query
109 return query
110
110
111 def get_cache_region(name, region):
111 def get_cache_region(name, region):
112 if region not in beaker.cache.cache_regions:
112 if region not in beaker.cache.cache_regions:
113 raise BeakerException('Cache region not configured: %s' % region)
113 raise BeakerException('Cache region not configured: %s'
114 'Check if proper cache settings are in the .ini files' % region)
114 kw = beaker.cache.cache_regions[region]
115 kw = beaker.cache.cache_regions[region]
115 return beaker.cache.Cache._get_cache(name, kw)
116 return beaker.cache.Cache._get_cache(name, kw)
116
117
117 def _get_cache_parameters(query):
118 def _get_cache_parameters(query):
118 """For a query with cache_region and cache_namespace configured,
119 """For a query with cache_region and cache_namespace configured,
119 return the correspoinding Cache instance and cache key, based
120 return the correspoinding Cache instance and cache key, based
120 on this query's current criterion and parameter values.
121 on this query's current criterion and parameter values.
121
122
122 """
123 """
123 if not hasattr(query, '_cache_parameters'):
124 if not hasattr(query, '_cache_parameters'):
124 raise ValueError("This Query does not have caching parameters configured.")
125 raise ValueError("This Query does not have caching parameters configured.")
125
126
126 region, namespace, cache_key = query._cache_parameters
127 region, namespace, cache_key = query._cache_parameters
127
128
128 namespace = _namespace_from_query(namespace, query)
129 namespace = _namespace_from_query(namespace, query)
129
130
130 if cache_key is None:
131 if cache_key is None:
131 # cache key - the value arguments from this query's parameters.
132 # cache key - the value arguments from this query's parameters.
132 args = _params_from_query(query)
133 args = _params_from_query(query)
133 cache_key = " ".join([str(x) for x in args])
134 cache_key = " ".join([str(x) for x in args])
134
135
135 # get cache
136 # get cache
136 #cache = query.cache_manager.get_cache_region(namespace, region)
137 #cache = query.cache_manager.get_cache_region(namespace, region)
137 cache = get_cache_region(namespace, region)
138 cache = get_cache_region(namespace, region)
138 # optional - hash the cache_key too for consistent length
139 # optional - hash the cache_key too for consistent length
139 # import uuid
140 # import uuid
140 # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
141 # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
141
142
142 return cache, cache_key
143 return cache, cache_key
143
144
144 def _namespace_from_query(namespace, query):
145 def _namespace_from_query(namespace, query):
145 # cache namespace - the token handed in by the
146 # cache namespace - the token handed in by the
146 # option + class we're querying against
147 # option + class we're querying against
147 namespace = " ".join([namespace] + [str(x) for x in query._entities])
148 namespace = " ".join([namespace] + [str(x) for x in query._entities])
148
149
149 # memcached wants this
150 # memcached wants this
150 namespace = namespace.replace(' ', '_')
151 namespace = namespace.replace(' ', '_')
151
152
152 return namespace
153 return namespace
153
154
154 def _set_cache_parameters(query, region, namespace, cache_key):
155 def _set_cache_parameters(query, region, namespace, cache_key):
155
156
156 if hasattr(query, '_cache_parameters'):
157 if hasattr(query, '_cache_parameters'):
157 region, namespace, cache_key = query._cache_parameters
158 region, namespace, cache_key = query._cache_parameters
158 raise ValueError("This query is already configured "
159 raise ValueError("This query is already configured "
159 "for region %r namespace %r" %
160 "for region %r namespace %r" %
160 (region, namespace)
161 (region, namespace)
161 )
162 )
162 query._cache_parameters = region, namespace, cache_key
163 query._cache_parameters = region, namespace, cache_key
163
164
164 class FromCache(MapperOption):
165 class FromCache(MapperOption):
165 """Specifies that a Query should load results from a cache."""
166 """Specifies that a Query should load results from a cache."""
166
167
167 propagate_to_loaders = False
168 propagate_to_loaders = False
168
169
169 def __init__(self, region, namespace, cache_key=None):
170 def __init__(self, region, namespace, cache_key=None):
170 """Construct a new FromCache.
171 """Construct a new FromCache.
171
172
172 :param region: the cache region. Should be a
173 :param region: the cache region. Should be a
173 region configured in the Beaker CacheManager.
174 region configured in the Beaker CacheManager.
174
175
175 :param namespace: the cache namespace. Should
176 :param namespace: the cache namespace. Should
176 be a name uniquely describing the target Query's
177 be a name uniquely describing the target Query's
177 lexical structure.
178 lexical structure.
178
179
179 :param cache_key: optional. A string cache key
180 :param cache_key: optional. A string cache key
180 that will serve as the key to the query. Use this
181 that will serve as the key to the query. Use this
181 if your query has a huge amount of parameters (such
182 if your query has a huge amount of parameters (such
182 as when using in_()) which correspond more simply to
183 as when using in_()) which correspond more simply to
183 some other identifier.
184 some other identifier.
184
185
185 """
186 """
186 self.region = region
187 self.region = region
187 self.namespace = namespace
188 self.namespace = namespace
188 self.cache_key = cache_key
189 self.cache_key = cache_key
189
190
190 def process_query(self, query):
191 def process_query(self, query):
191 """Process a Query during normal loading operation."""
192 """Process a Query during normal loading operation."""
192
193
193 _set_cache_parameters(query, self.region, self.namespace, self.cache_key)
194 _set_cache_parameters(query, self.region, self.namespace, self.cache_key)
194
195
195 class RelationshipCache(MapperOption):
196 class RelationshipCache(MapperOption):
196 """Specifies that a Query as called within a "lazy load"
197 """Specifies that a Query as called within a "lazy load"
197 should load results from a cache."""
198 should load results from a cache."""
198
199
199 propagate_to_loaders = True
200 propagate_to_loaders = True
200
201
201 def __init__(self, region, namespace, attribute):
202 def __init__(self, region, namespace, attribute):
202 """Construct a new RelationshipCache.
203 """Construct a new RelationshipCache.
203
204
204 :param region: the cache region. Should be a
205 :param region: the cache region. Should be a
205 region configured in the Beaker CacheManager.
206 region configured in the Beaker CacheManager.
206
207
207 :param namespace: the cache namespace. Should
208 :param namespace: the cache namespace. Should
208 be a name uniquely describing the target Query's
209 be a name uniquely describing the target Query's
209 lexical structure.
210 lexical structure.
210
211
211 :param attribute: A Class.attribute which
212 :param attribute: A Class.attribute which
212 indicates a particular class relationship() whose
213 indicates a particular class relationship() whose
213 lazy loader should be pulled from the cache.
214 lazy loader should be pulled from the cache.
214
215
215 """
216 """
216 self.region = region
217 self.region = region
217 self.namespace = namespace
218 self.namespace = namespace
218 self._relationship_options = {
219 self._relationship_options = {
219 (attribute.property.parent.class_, attribute.property.key) : self
220 (attribute.property.parent.class_, attribute.property.key) : self
220 }
221 }
221
222
222 def process_query_conditionally(self, query):
223 def process_query_conditionally(self, query):
223 """Process a Query that is used within a lazy loader.
224 """Process a Query that is used within a lazy loader.
224
225
225 (the process_query_conditionally() method is a SQLAlchemy
226 (the process_query_conditionally() method is a SQLAlchemy
226 hook invoked only within lazyload.)
227 hook invoked only within lazyload.)
227
228
228 """
229 """
229 if query._current_path:
230 if query._current_path:
230 mapper, key = query._current_path[-2:]
231 mapper, key = query._current_path[-2:]
231
232
232 for cls in mapper.class_.__mro__:
233 for cls in mapper.class_.__mro__:
233 if (cls, key) in self._relationship_options:
234 if (cls, key) in self._relationship_options:
234 relationship_option = self._relationship_options[(cls, key)]
235 relationship_option = self._relationship_options[(cls, key)]
235 _set_cache_parameters(
236 _set_cache_parameters(
236 query,
237 query,
237 relationship_option.region,
238 relationship_option.region,
238 relationship_option.namespace,
239 relationship_option.namespace,
239 None)
240 None)
240
241
241 def and_(self, option):
242 def and_(self, option):
242 """Chain another RelationshipCache option to this one.
243 """Chain another RelationshipCache option to this one.
243
244
244 While many RelationshipCache objects can be specified on a single
245 While many RelationshipCache objects can be specified on a single
245 Query separately, chaining them together allows for a more efficient
246 Query separately, chaining them together allows for a more efficient
246 lookup during load.
247 lookup during load.
247
248
248 """
249 """
249 self._relationship_options.update(option._relationship_options)
250 self._relationship_options.update(option._relationship_options)
250 return self
251 return self
251
252
252
253
253 def _params_from_query(query):
254 def _params_from_query(query):
254 """Pull the bind parameter values from a query.
255 """Pull the bind parameter values from a query.
255
256
256 This takes into account any scalar attribute bindparam set up.
257 This takes into account any scalar attribute bindparam set up.
257
258
258 E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7)))
259 E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7)))
259 would return [5, 7].
260 would return [5, 7].
260
261
261 """
262 """
262 v = []
263 v = []
263 def visit_bindparam(bind):
264 def visit_bindparam(bind):
264 value = query._params.get(bind.key, bind.value)
265 value = query._params.get(bind.key, bind.value)
265
266
266 # lazyloader may dig a callable in here, intended
267 # lazyloader may dig a callable in here, intended
267 # to late-evaluate params after autoflush is called.
268 # to late-evaluate params after autoflush is called.
268 # convert to a scalar value.
269 # convert to a scalar value.
269 if callable(value):
270 if callable(value):
270 value = value()
271 value = value()
271
272
272 v.append(value)
273 v.append(value)
273 if query._criterion is not None:
274 if query._criterion is not None:
274 visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam})
275 visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam})
275 return v
276 return v
General Comments 0
You need to be logged in to leave comments. Login now