Show More
@@ -1,275 +1,276 | |||||
1 | """caching_query.py |
|
1 | """caching_query.py | |
2 |
|
2 | |||
3 | Represent persistence structures which allow the usage of |
|
3 | Represent persistence structures which allow the usage of | |
4 | Beaker caching with SQLAlchemy. |
|
4 | Beaker caching with SQLAlchemy. | |
5 |
|
5 | |||
6 | The three new concepts introduced here are: |
|
6 | The three new concepts introduced here are: | |
7 |
|
7 | |||
8 | * CachingQuery - a Query subclass that caches and |
|
8 | * CachingQuery - a Query subclass that caches and | |
9 | retrieves results in/from Beaker. |
|
9 | retrieves results in/from Beaker. | |
10 | * FromCache - a query option that establishes caching |
|
10 | * FromCache - a query option that establishes caching | |
11 | parameters on a Query |
|
11 | parameters on a Query | |
12 | * RelationshipCache - a variant of FromCache which is specific |
|
12 | * RelationshipCache - a variant of FromCache which is specific | |
13 | to a query invoked during a lazy load. |
|
13 | to a query invoked during a lazy load. | |
14 | * _params_from_query - extracts value parameters from |
|
14 | * _params_from_query - extracts value parameters from | |
15 | a Query. |
|
15 | a Query. | |
16 |
|
16 | |||
17 | The rest of what's here are standard SQLAlchemy and |
|
17 | The rest of what's here are standard SQLAlchemy and | |
18 | Beaker constructs. |
|
18 | Beaker constructs. | |
19 |
|
19 | |||
20 | """ |
|
20 | """ | |
21 | from beaker.exceptions import BeakerException |
|
21 | from beaker.exceptions import BeakerException | |
22 | from sqlalchemy.orm.interfaces import MapperOption |
|
22 | from sqlalchemy.orm.interfaces import MapperOption | |
23 | from sqlalchemy.orm.query import Query |
|
23 | from sqlalchemy.orm.query import Query | |
24 | from sqlalchemy.sql import visitors |
|
24 | from sqlalchemy.sql import visitors | |
25 | import beaker |
|
25 | import beaker | |
26 |
|
26 | |||
27 | class CachingQuery(Query): |
|
27 | class CachingQuery(Query): | |
28 | """A Query subclass which optionally loads full results from a Beaker |
|
28 | """A Query subclass which optionally loads full results from a Beaker | |
29 | cache region. |
|
29 | cache region. | |
30 |
|
30 | |||
31 | The CachingQuery stores additional state that allows it to consult |
|
31 | The CachingQuery stores additional state that allows it to consult | |
32 | a Beaker cache before accessing the database: |
|
32 | a Beaker cache before accessing the database: | |
33 |
|
33 | |||
34 | * A "region", which is a cache region argument passed to a |
|
34 | * A "region", which is a cache region argument passed to a | |
35 | Beaker CacheManager, specifies a particular cache configuration |
|
35 | Beaker CacheManager, specifies a particular cache configuration | |
36 | (including backend implementation, expiration times, etc.) |
|
36 | (including backend implementation, expiration times, etc.) | |
37 | * A "namespace", which is a qualifying name that identifies a |
|
37 | * A "namespace", which is a qualifying name that identifies a | |
38 | group of keys within the cache. A query that filters on a name |
|
38 | group of keys within the cache. A query that filters on a name | |
39 | might use the name "by_name", a query that filters on a date range |
|
39 | might use the name "by_name", a query that filters on a date range | |
40 | to a joined table might use the name "related_date_range". |
|
40 | to a joined table might use the name "related_date_range". | |
41 |
|
41 | |||
42 | When the above state is present, a Beaker cache is retrieved. |
|
42 | When the above state is present, a Beaker cache is retrieved. | |
43 |
|
43 | |||
44 | The "namespace" name is first concatenated with |
|
44 | The "namespace" name is first concatenated with | |
45 | a string composed of the individual entities and columns the Query |
|
45 | a string composed of the individual entities and columns the Query | |
46 | requests, i.e. such as ``Query(User.id, User.name)``. |
|
46 | requests, i.e. such as ``Query(User.id, User.name)``. | |
47 |
|
47 | |||
48 | The Beaker cache is then loaded from the cache manager based |
|
48 | The Beaker cache is then loaded from the cache manager based | |
49 | on the region and composed namespace. The key within the cache |
|
49 | on the region and composed namespace. The key within the cache | |
50 | itself is then constructed against the bind parameters specified |
|
50 | itself is then constructed against the bind parameters specified | |
51 | by this query, which are usually literals defined in the |
|
51 | by this query, which are usually literals defined in the | |
52 | WHERE clause. |
|
52 | WHERE clause. | |
53 |
|
53 | |||
54 | The FromCache and RelationshipCache mapper options below represent |
|
54 | The FromCache and RelationshipCache mapper options below represent | |
55 | the "public" method of configuring this state upon the CachingQuery. |
|
55 | the "public" method of configuring this state upon the CachingQuery. | |
56 |
|
56 | |||
57 | """ |
|
57 | """ | |
58 |
|
58 | |||
59 | def __init__(self, manager, *args, **kw): |
|
59 | def __init__(self, manager, *args, **kw): | |
60 | self.cache_manager = manager |
|
60 | self.cache_manager = manager | |
61 | Query.__init__(self, *args, **kw) |
|
61 | Query.__init__(self, *args, **kw) | |
62 |
|
62 | |||
63 | def __iter__(self): |
|
63 | def __iter__(self): | |
64 | """override __iter__ to pull results from Beaker |
|
64 | """override __iter__ to pull results from Beaker | |
65 | if particular attributes have been configured. |
|
65 | if particular attributes have been configured. | |
66 |
|
66 | |||
67 | Note that this approach does *not* detach the loaded objects from |
|
67 | Note that this approach does *not* detach the loaded objects from | |
68 | the current session. If the cache backend is an in-process cache |
|
68 | the current session. If the cache backend is an in-process cache | |
69 | (like "memory") and lives beyond the scope of the current session's |
|
69 | (like "memory") and lives beyond the scope of the current session's | |
70 | transaction, those objects may be expired. The method here can be |
|
70 | transaction, those objects may be expired. The method here can be | |
71 | modified to first expunge() each loaded item from the current |
|
71 | modified to first expunge() each loaded item from the current | |
72 | session before returning the list of items, so that the items |
|
72 | session before returning the list of items, so that the items | |
73 | in the cache are not the same ones in the current Session. |
|
73 | in the cache are not the same ones in the current Session. | |
74 |
|
74 | |||
75 | """ |
|
75 | """ | |
76 | if hasattr(self, '_cache_parameters'): |
|
76 | if hasattr(self, '_cache_parameters'): | |
77 | return self.get_value(createfunc=lambda: list(Query.__iter__(self))) |
|
77 | return self.get_value(createfunc=lambda: list(Query.__iter__(self))) | |
78 | else: |
|
78 | else: | |
79 | return Query.__iter__(self) |
|
79 | return Query.__iter__(self) | |
80 |
|
80 | |||
81 | def invalidate(self): |
|
81 | def invalidate(self): | |
82 | """Invalidate the value represented by this Query.""" |
|
82 | """Invalidate the value represented by this Query.""" | |
83 |
|
83 | |||
84 | cache, cache_key = _get_cache_parameters(self) |
|
84 | cache, cache_key = _get_cache_parameters(self) | |
85 | cache.remove(cache_key) |
|
85 | cache.remove(cache_key) | |
86 |
|
86 | |||
87 | def get_value(self, merge=True, createfunc=None): |
|
87 | def get_value(self, merge=True, createfunc=None): | |
88 | """Return the value from the cache for this query. |
|
88 | """Return the value from the cache for this query. | |
89 |
|
89 | |||
90 | Raise KeyError if no value present and no |
|
90 | Raise KeyError if no value present and no | |
91 | createfunc specified. |
|
91 | createfunc specified. | |
92 |
|
92 | |||
93 | """ |
|
93 | """ | |
94 | cache, cache_key = _get_cache_parameters(self) |
|
94 | cache, cache_key = _get_cache_parameters(self) | |
95 | ret = cache.get_value(cache_key, createfunc=createfunc) |
|
95 | ret = cache.get_value(cache_key, createfunc=createfunc) | |
96 | if merge: |
|
96 | if merge: | |
97 | ret = self.merge_result(ret, load=False) |
|
97 | ret = self.merge_result(ret, load=False) | |
98 | return ret |
|
98 | return ret | |
99 |
|
99 | |||
100 | def set_value(self, value): |
|
100 | def set_value(self, value): | |
101 | """Set the value in the cache for this query.""" |
|
101 | """Set the value in the cache for this query.""" | |
102 |
|
102 | |||
103 | cache, cache_key = _get_cache_parameters(self) |
|
103 | cache, cache_key = _get_cache_parameters(self) | |
104 | cache.put(cache_key, value) |
|
104 | cache.put(cache_key, value) | |
105 |
|
105 | |||
106 | def query_callable(manager): |
|
106 | def query_callable(manager): | |
107 | def query(*arg, **kw): |
|
107 | def query(*arg, **kw): | |
108 | return CachingQuery(manager, *arg, **kw) |
|
108 | return CachingQuery(manager, *arg, **kw) | |
109 | return query |
|
109 | return query | |
110 |
|
110 | |||
111 | def get_cache_region(name, region): |
|
111 | def get_cache_region(name, region): | |
112 |
|
|
112 | if region not in beaker.cache.cache_regions: | |
113 |
|
|
113 | raise BeakerException('Cache region not configured: %s' | |
|
114 | 'Check if proper cache settings are in the .ini files' % region) | |||
114 |
|
|
115 | kw = beaker.cache.cache_regions[region] | |
115 |
|
|
116 | return beaker.cache.Cache._get_cache(name, kw) | |
116 |
|
|
117 | ||
117 | def _get_cache_parameters(query): |
|
118 | def _get_cache_parameters(query): | |
118 | """For a query with cache_region and cache_namespace configured, |
|
119 | """For a query with cache_region and cache_namespace configured, | |
119 | return the correspoinding Cache instance and cache key, based |
|
120 | return the correspoinding Cache instance and cache key, based | |
120 | on this query's current criterion and parameter values. |
|
121 | on this query's current criterion and parameter values. | |
121 |
|
122 | |||
122 | """ |
|
123 | """ | |
123 | if not hasattr(query, '_cache_parameters'): |
|
124 | if not hasattr(query, '_cache_parameters'): | |
124 | raise ValueError("This Query does not have caching parameters configured.") |
|
125 | raise ValueError("This Query does not have caching parameters configured.") | |
125 |
|
126 | |||
126 | region, namespace, cache_key = query._cache_parameters |
|
127 | region, namespace, cache_key = query._cache_parameters | |
127 |
|
128 | |||
128 | namespace = _namespace_from_query(namespace, query) |
|
129 | namespace = _namespace_from_query(namespace, query) | |
129 |
|
130 | |||
130 | if cache_key is None: |
|
131 | if cache_key is None: | |
131 | # cache key - the value arguments from this query's parameters. |
|
132 | # cache key - the value arguments from this query's parameters. | |
132 | args = _params_from_query(query) |
|
133 | args = _params_from_query(query) | |
133 | cache_key = " ".join([str(x) for x in args]) |
|
134 | cache_key = " ".join([str(x) for x in args]) | |
134 |
|
135 | |||
135 | # get cache |
|
136 | # get cache | |
136 | #cache = query.cache_manager.get_cache_region(namespace, region) |
|
137 | #cache = query.cache_manager.get_cache_region(namespace, region) | |
137 | cache = get_cache_region(namespace, region) |
|
138 | cache = get_cache_region(namespace, region) | |
138 | # optional - hash the cache_key too for consistent length |
|
139 | # optional - hash the cache_key too for consistent length | |
139 | # import uuid |
|
140 | # import uuid | |
140 | # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) |
|
141 | # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) | |
141 |
|
142 | |||
142 | return cache, cache_key |
|
143 | return cache, cache_key | |
143 |
|
144 | |||
144 | def _namespace_from_query(namespace, query): |
|
145 | def _namespace_from_query(namespace, query): | |
145 | # cache namespace - the token handed in by the |
|
146 | # cache namespace - the token handed in by the | |
146 | # option + class we're querying against |
|
147 | # option + class we're querying against | |
147 | namespace = " ".join([namespace] + [str(x) for x in query._entities]) |
|
148 | namespace = " ".join([namespace] + [str(x) for x in query._entities]) | |
148 |
|
149 | |||
149 | # memcached wants this |
|
150 | # memcached wants this | |
150 | namespace = namespace.replace(' ', '_') |
|
151 | namespace = namespace.replace(' ', '_') | |
151 |
|
152 | |||
152 | return namespace |
|
153 | return namespace | |
153 |
|
154 | |||
154 | def _set_cache_parameters(query, region, namespace, cache_key): |
|
155 | def _set_cache_parameters(query, region, namespace, cache_key): | |
155 |
|
156 | |||
156 | if hasattr(query, '_cache_parameters'): |
|
157 | if hasattr(query, '_cache_parameters'): | |
157 | region, namespace, cache_key = query._cache_parameters |
|
158 | region, namespace, cache_key = query._cache_parameters | |
158 | raise ValueError("This query is already configured " |
|
159 | raise ValueError("This query is already configured " | |
159 | "for region %r namespace %r" % |
|
160 | "for region %r namespace %r" % | |
160 | (region, namespace) |
|
161 | (region, namespace) | |
161 | ) |
|
162 | ) | |
162 | query._cache_parameters = region, namespace, cache_key |
|
163 | query._cache_parameters = region, namespace, cache_key | |
163 |
|
164 | |||
164 | class FromCache(MapperOption): |
|
165 | class FromCache(MapperOption): | |
165 | """Specifies that a Query should load results from a cache.""" |
|
166 | """Specifies that a Query should load results from a cache.""" | |
166 |
|
167 | |||
167 | propagate_to_loaders = False |
|
168 | propagate_to_loaders = False | |
168 |
|
169 | |||
169 | def __init__(self, region, namespace, cache_key=None): |
|
170 | def __init__(self, region, namespace, cache_key=None): | |
170 | """Construct a new FromCache. |
|
171 | """Construct a new FromCache. | |
171 |
|
172 | |||
172 | :param region: the cache region. Should be a |
|
173 | :param region: the cache region. Should be a | |
173 | region configured in the Beaker CacheManager. |
|
174 | region configured in the Beaker CacheManager. | |
174 |
|
175 | |||
175 | :param namespace: the cache namespace. Should |
|
176 | :param namespace: the cache namespace. Should | |
176 | be a name uniquely describing the target Query's |
|
177 | be a name uniquely describing the target Query's | |
177 | lexical structure. |
|
178 | lexical structure. | |
178 |
|
179 | |||
179 | :param cache_key: optional. A string cache key |
|
180 | :param cache_key: optional. A string cache key | |
180 | that will serve as the key to the query. Use this |
|
181 | that will serve as the key to the query. Use this | |
181 | if your query has a huge amount of parameters (such |
|
182 | if your query has a huge amount of parameters (such | |
182 | as when using in_()) which correspond more simply to |
|
183 | as when using in_()) which correspond more simply to | |
183 | some other identifier. |
|
184 | some other identifier. | |
184 |
|
185 | |||
185 | """ |
|
186 | """ | |
186 | self.region = region |
|
187 | self.region = region | |
187 | self.namespace = namespace |
|
188 | self.namespace = namespace | |
188 | self.cache_key = cache_key |
|
189 | self.cache_key = cache_key | |
189 |
|
190 | |||
190 | def process_query(self, query): |
|
191 | def process_query(self, query): | |
191 | """Process a Query during normal loading operation.""" |
|
192 | """Process a Query during normal loading operation.""" | |
192 |
|
193 | |||
193 | _set_cache_parameters(query, self.region, self.namespace, self.cache_key) |
|
194 | _set_cache_parameters(query, self.region, self.namespace, self.cache_key) | |
194 |
|
195 | |||
195 | class RelationshipCache(MapperOption): |
|
196 | class RelationshipCache(MapperOption): | |
196 | """Specifies that a Query as called within a "lazy load" |
|
197 | """Specifies that a Query as called within a "lazy load" | |
197 | should load results from a cache.""" |
|
198 | should load results from a cache.""" | |
198 |
|
199 | |||
199 | propagate_to_loaders = True |
|
200 | propagate_to_loaders = True | |
200 |
|
201 | |||
201 | def __init__(self, region, namespace, attribute): |
|
202 | def __init__(self, region, namespace, attribute): | |
202 | """Construct a new RelationshipCache. |
|
203 | """Construct a new RelationshipCache. | |
203 |
|
204 | |||
204 | :param region: the cache region. Should be a |
|
205 | :param region: the cache region. Should be a | |
205 | region configured in the Beaker CacheManager. |
|
206 | region configured in the Beaker CacheManager. | |
206 |
|
207 | |||
207 | :param namespace: the cache namespace. Should |
|
208 | :param namespace: the cache namespace. Should | |
208 | be a name uniquely describing the target Query's |
|
209 | be a name uniquely describing the target Query's | |
209 | lexical structure. |
|
210 | lexical structure. | |
210 |
|
211 | |||
211 | :param attribute: A Class.attribute which |
|
212 | :param attribute: A Class.attribute which | |
212 | indicates a particular class relationship() whose |
|
213 | indicates a particular class relationship() whose | |
213 | lazy loader should be pulled from the cache. |
|
214 | lazy loader should be pulled from the cache. | |
214 |
|
215 | |||
215 | """ |
|
216 | """ | |
216 | self.region = region |
|
217 | self.region = region | |
217 | self.namespace = namespace |
|
218 | self.namespace = namespace | |
218 | self._relationship_options = { |
|
219 | self._relationship_options = { | |
219 | (attribute.property.parent.class_, attribute.property.key) : self |
|
220 | (attribute.property.parent.class_, attribute.property.key) : self | |
220 | } |
|
221 | } | |
221 |
|
222 | |||
222 | def process_query_conditionally(self, query): |
|
223 | def process_query_conditionally(self, query): | |
223 | """Process a Query that is used within a lazy loader. |
|
224 | """Process a Query that is used within a lazy loader. | |
224 |
|
225 | |||
225 | (the process_query_conditionally() method is a SQLAlchemy |
|
226 | (the process_query_conditionally() method is a SQLAlchemy | |
226 | hook invoked only within lazyload.) |
|
227 | hook invoked only within lazyload.) | |
227 |
|
228 | |||
228 | """ |
|
229 | """ | |
229 | if query._current_path: |
|
230 | if query._current_path: | |
230 | mapper, key = query._current_path[-2:] |
|
231 | mapper, key = query._current_path[-2:] | |
231 |
|
232 | |||
232 | for cls in mapper.class_.__mro__: |
|
233 | for cls in mapper.class_.__mro__: | |
233 | if (cls, key) in self._relationship_options: |
|
234 | if (cls, key) in self._relationship_options: | |
234 | relationship_option = self._relationship_options[(cls, key)] |
|
235 | relationship_option = self._relationship_options[(cls, key)] | |
235 | _set_cache_parameters( |
|
236 | _set_cache_parameters( | |
236 | query, |
|
237 | query, | |
237 | relationship_option.region, |
|
238 | relationship_option.region, | |
238 | relationship_option.namespace, |
|
239 | relationship_option.namespace, | |
239 | None) |
|
240 | None) | |
240 |
|
241 | |||
241 | def and_(self, option): |
|
242 | def and_(self, option): | |
242 | """Chain another RelationshipCache option to this one. |
|
243 | """Chain another RelationshipCache option to this one. | |
243 |
|
244 | |||
244 | While many RelationshipCache objects can be specified on a single |
|
245 | While many RelationshipCache objects can be specified on a single | |
245 | Query separately, chaining them together allows for a more efficient |
|
246 | Query separately, chaining them together allows for a more efficient | |
246 | lookup during load. |
|
247 | lookup during load. | |
247 |
|
248 | |||
248 | """ |
|
249 | """ | |
249 | self._relationship_options.update(option._relationship_options) |
|
250 | self._relationship_options.update(option._relationship_options) | |
250 | return self |
|
251 | return self | |
251 |
|
252 | |||
252 |
|
253 | |||
253 | def _params_from_query(query): |
|
254 | def _params_from_query(query): | |
254 | """Pull the bind parameter values from a query. |
|
255 | """Pull the bind parameter values from a query. | |
255 |
|
256 | |||
256 | This takes into account any scalar attribute bindparam set up. |
|
257 | This takes into account any scalar attribute bindparam set up. | |
257 |
|
258 | |||
258 | E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) |
|
259 | E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) | |
259 | would return [5, 7]. |
|
260 | would return [5, 7]. | |
260 |
|
261 | |||
261 | """ |
|
262 | """ | |
262 | v = [] |
|
263 | v = [] | |
263 | def visit_bindparam(bind): |
|
264 | def visit_bindparam(bind): | |
264 | value = query._params.get(bind.key, bind.value) |
|
265 | value = query._params.get(bind.key, bind.value) | |
265 |
|
266 | |||
266 | # lazyloader may dig a callable in here, intended |
|
267 | # lazyloader may dig a callable in here, intended | |
267 | # to late-evaluate params after autoflush is called. |
|
268 | # to late-evaluate params after autoflush is called. | |
268 | # convert to a scalar value. |
|
269 | # convert to a scalar value. | |
269 | if callable(value): |
|
270 | if callable(value): | |
270 | value = value() |
|
271 | value = value() | |
271 |
|
272 | |||
272 | v.append(value) |
|
273 | v.append(value) | |
273 | if query._criterion is not None: |
|
274 | if query._criterion is not None: | |
274 | visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) |
|
275 | visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) | |
275 | return v |
|
276 | return v |
General Comments 0
You need to be logged in to leave comments.
Login now