Show More
@@ -1,276 +1,276 b'' | |||||
1 | """caching_query.py |
|
1 | """caching_query.py | |
2 |
|
2 | |||
3 | Represent persistence structures which allow the usage of |
|
3 | Represent persistence structures which allow the usage of | |
4 | Beaker caching with SQLAlchemy. |
|
4 | Beaker caching with SQLAlchemy. | |
5 |
|
5 | |||
6 | The three new concepts introduced here are: |
|
6 | The three new concepts introduced here are: | |
7 |
|
7 | |||
8 | * CachingQuery - a Query subclass that caches and |
|
8 | * CachingQuery - a Query subclass that caches and | |
9 | retrieves results in/from Beaker. |
|
9 | retrieves results in/from Beaker. | |
10 | * FromCache - a query option that establishes caching |
|
10 | * FromCache - a query option that establishes caching | |
11 | parameters on a Query |
|
11 | parameters on a Query | |
12 | * RelationshipCache - a variant of FromCache which is specific |
|
12 | * RelationshipCache - a variant of FromCache which is specific | |
13 | to a query invoked during a lazy load. |
|
13 | to a query invoked during a lazy load. | |
14 | * _params_from_query - extracts value parameters from |
|
14 | * _params_from_query - extracts value parameters from | |
15 | a Query. |
|
15 | a Query. | |
16 |
|
16 | |||
17 | The rest of what's here are standard SQLAlchemy and |
|
17 | The rest of what's here are standard SQLAlchemy and | |
18 | Beaker constructs. |
|
18 | Beaker constructs. | |
19 |
|
19 | |||
20 | """ |
|
20 | """ | |
21 | from beaker.exceptions import BeakerException |
|
21 | from beaker.exceptions import BeakerException | |
22 | from sqlalchemy.orm.interfaces import MapperOption |
|
22 | from sqlalchemy.orm.interfaces import MapperOption | |
23 | from sqlalchemy.orm.query import Query |
|
23 | from sqlalchemy.orm.query import Query | |
24 | from sqlalchemy.sql import visitors |
|
24 | from sqlalchemy.sql import visitors | |
25 | import beaker |
|
25 | import beaker | |
26 |
|
26 | |||
27 | class CachingQuery(Query): |
|
27 | class CachingQuery(Query): | |
28 | """A Query subclass which optionally loads full results from a Beaker |
|
28 | """A Query subclass which optionally loads full results from a Beaker | |
29 | cache region. |
|
29 | cache region. | |
30 |
|
30 | |||
31 | The CachingQuery stores additional state that allows it to consult |
|
31 | The CachingQuery stores additional state that allows it to consult | |
32 | a Beaker cache before accessing the database: |
|
32 | a Beaker cache before accessing the database: | |
33 |
|
33 | |||
34 | * A "region", which is a cache region argument passed to a |
|
34 | * A "region", which is a cache region argument passed to a | |
35 | Beaker CacheManager, specifies a particular cache configuration |
|
35 | Beaker CacheManager, specifies a particular cache configuration | |
36 | (including backend implementation, expiration times, etc.) |
|
36 | (including backend implementation, expiration times, etc.) | |
37 | * A "namespace", which is a qualifying name that identifies a |
|
37 | * A "namespace", which is a qualifying name that identifies a | |
38 | group of keys within the cache. A query that filters on a name |
|
38 | group of keys within the cache. A query that filters on a name | |
39 | might use the name "by_name", a query that filters on a date range |
|
39 | might use the name "by_name", a query that filters on a date range | |
40 | to a joined table might use the name "related_date_range". |
|
40 | to a joined table might use the name "related_date_range". | |
41 |
|
41 | |||
42 | When the above state is present, a Beaker cache is retrieved. |
|
42 | When the above state is present, a Beaker cache is retrieved. | |
43 |
|
43 | |||
44 | The "namespace" name is first concatenated with |
|
44 | The "namespace" name is first concatenated with | |
45 | a string composed of the individual entities and columns the Query |
|
45 | a string composed of the individual entities and columns the Query | |
46 | requests, i.e. such as ``Query(User.id, User.name)``. |
|
46 | requests, i.e. such as ``Query(User.id, User.name)``. | |
47 |
|
47 | |||
48 | The Beaker cache is then loaded from the cache manager based |
|
48 | The Beaker cache is then loaded from the cache manager based | |
49 | on the region and composed namespace. The key within the cache |
|
49 | on the region and composed namespace. The key within the cache | |
50 | itself is then constructed against the bind parameters specified |
|
50 | itself is then constructed against the bind parameters specified | |
51 | by this query, which are usually literals defined in the |
|
51 | by this query, which are usually literals defined in the | |
52 | WHERE clause. |
|
52 | WHERE clause. | |
53 |
|
53 | |||
54 | The FromCache and RelationshipCache mapper options below represent |
|
54 | The FromCache and RelationshipCache mapper options below represent | |
55 | the "public" method of configuring this state upon the CachingQuery. |
|
55 | the "public" method of configuring this state upon the CachingQuery. | |
56 |
|
56 | |||
57 | """ |
|
57 | """ | |
58 |
|
58 | |||
59 | def __init__(self, manager, *args, **kw): |
|
59 | def __init__(self, manager, *args, **kw): | |
60 | self.cache_manager = manager |
|
60 | self.cache_manager = manager | |
61 | Query.__init__(self, *args, **kw) |
|
61 | Query.__init__(self, *args, **kw) | |
62 |
|
62 | |||
63 | def __iter__(self): |
|
63 | def __iter__(self): | |
64 | """override __iter__ to pull results from Beaker |
|
64 | """override __iter__ to pull results from Beaker | |
65 | if particular attributes have been configured. |
|
65 | if particular attributes have been configured. | |
66 |
|
66 | |||
67 | Note that this approach does *not* detach the loaded objects from |
|
67 | Note that this approach does *not* detach the loaded objects from | |
68 | the current session. If the cache backend is an in-process cache |
|
68 | the current session. If the cache backend is an in-process cache | |
69 | (like "memory") and lives beyond the scope of the current session's |
|
69 | (like "memory") and lives beyond the scope of the current session's | |
70 | transaction, those objects may be expired. The method here can be |
|
70 | transaction, those objects may be expired. The method here can be | |
71 | modified to first expunge() each loaded item from the current |
|
71 | modified to first expunge() each loaded item from the current | |
72 | session before returning the list of items, so that the items |
|
72 | session before returning the list of items, so that the items | |
73 | in the cache are not the same ones in the current Session. |
|
73 | in the cache are not the same ones in the current Session. | |
74 |
|
74 | |||
75 | """ |
|
75 | """ | |
76 | if hasattr(self, '_cache_parameters'): |
|
76 | if hasattr(self, '_cache_parameters'): | |
77 | return self.get_value(createfunc=lambda: list(Query.__iter__(self))) |
|
77 | return self.get_value(createfunc=lambda: list(Query.__iter__(self))) | |
78 | else: |
|
78 | else: | |
79 | return Query.__iter__(self) |
|
79 | return Query.__iter__(self) | |
80 |
|
80 | |||
81 | def invalidate(self): |
|
81 | def invalidate(self): | |
82 | """Invalidate the value represented by this Query.""" |
|
82 | """Invalidate the value represented by this Query.""" | |
83 |
|
83 | |||
84 | cache, cache_key = _get_cache_parameters(self) |
|
84 | cache, cache_key = _get_cache_parameters(self) | |
85 | cache.remove(cache_key) |
|
85 | cache.remove(cache_key) | |
86 |
|
86 | |||
87 | def get_value(self, merge=True, createfunc=None): |
|
87 | def get_value(self, merge=True, createfunc=None): | |
88 | """Return the value from the cache for this query. |
|
88 | """Return the value from the cache for this query. | |
89 |
|
89 | |||
90 | Raise KeyError if no value present and no |
|
90 | Raise KeyError if no value present and no | |
91 | createfunc specified. |
|
91 | createfunc specified. | |
92 |
|
92 | |||
93 | """ |
|
93 | """ | |
94 | cache, cache_key = _get_cache_parameters(self) |
|
94 | cache, cache_key = _get_cache_parameters(self) | |
95 | ret = cache.get_value(cache_key, createfunc=createfunc) |
|
95 | ret = cache.get_value(cache_key, createfunc=createfunc) | |
96 | if merge: |
|
96 | if merge: | |
97 | ret = self.merge_result(ret, load=False) |
|
97 | ret = self.merge_result(ret, load=False) | |
98 | return ret |
|
98 | return ret | |
99 |
|
99 | |||
100 | def set_value(self, value): |
|
100 | def set_value(self, value): | |
101 | """Set the value in the cache for this query.""" |
|
101 | """Set the value in the cache for this query.""" | |
102 |
|
102 | |||
103 | cache, cache_key = _get_cache_parameters(self) |
|
103 | cache, cache_key = _get_cache_parameters(self) | |
104 | cache.put(cache_key, value) |
|
104 | cache.put(cache_key, value) | |
105 |
|
105 | |||
106 | def query_callable(manager): |
|
106 | def query_callable(manager): | |
107 | def query(*arg, **kw): |
|
107 | def query(*arg, **kw): | |
108 | return CachingQuery(manager, *arg, **kw) |
|
108 | return CachingQuery(manager, *arg, **kw) | |
109 | return query |
|
109 | return query | |
110 |
|
110 | |||
111 | def get_cache_region(name, region): |
|
111 | def get_cache_region(name, region): | |
112 | if region not in beaker.cache.cache_regions: |
|
112 | if region not in beaker.cache.cache_regions: | |
113 | raise BeakerException('Cache region `%s` not configured ' |
|
113 | raise BeakerException('Cache region `%s` not configured ' | |
114 | 'Check if proper cache settings are in the .ini files' % region) |
|
114 | 'Check if proper cache settings are in the .ini files' % region) | |
115 | kw = beaker.cache.cache_regions[region] |
|
115 | kw = beaker.cache.cache_regions[region] | |
116 | return beaker.cache.Cache._get_cache(name, kw) |
|
116 | return beaker.cache.Cache._get_cache(name, kw) | |
117 |
|
117 | |||
118 | def _get_cache_parameters(query): |
|
118 | def _get_cache_parameters(query): | |
119 | """For a query with cache_region and cache_namespace configured, |
|
119 | """For a query with cache_region and cache_namespace configured, | |
120 | return the correspoinding Cache instance and cache key, based |
|
120 | return the correspoinding Cache instance and cache key, based | |
121 | on this query's current criterion and parameter values. |
|
121 | on this query's current criterion and parameter values. | |
122 |
|
122 | |||
123 | """ |
|
123 | """ | |
124 | if not hasattr(query, '_cache_parameters'): |
|
124 | if not hasattr(query, '_cache_parameters'): | |
125 | raise ValueError("This Query does not have caching parameters configured.") |
|
125 | raise ValueError("This Query does not have caching parameters configured.") | |
126 |
|
126 | |||
127 | region, namespace, cache_key = query._cache_parameters |
|
127 | region, namespace, cache_key = query._cache_parameters | |
128 |
|
128 | |||
129 | namespace = _namespace_from_query(namespace, query) |
|
129 | namespace = _namespace_from_query(namespace, query) | |
130 |
|
130 | |||
131 | if cache_key is None: |
|
131 | if cache_key is None: | |
132 | # cache key - the value arguments from this query's parameters. |
|
132 | # cache key - the value arguments from this query's parameters. | |
133 | args = _params_from_query(query) |
|
133 | args = _params_from_query(query) | |
134 | cache_key = " ".join([str(x) for x in args]) |
|
134 | cache_key = " ".join([str(x) for x in args]) | |
135 |
|
135 | |||
136 | # get cache |
|
136 | # get cache | |
137 | #cache = query.cache_manager.get_cache_region(namespace, region) |
|
137 | #cache = query.cache_manager.get_cache_region(namespace, region) | |
138 | cache = get_cache_region(namespace, region) |
|
138 | cache = get_cache_region(namespace, region) | |
139 | # optional - hash the cache_key too for consistent length |
|
139 | # optional - hash the cache_key too for consistent length | |
140 | # import uuid |
|
140 | # import uuid | |
141 | # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) |
|
141 | # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) | |
142 |
|
142 | |||
143 | return cache, cache_key |
|
143 | return cache, cache_key | |
144 |
|
144 | |||
145 | def _namespace_from_query(namespace, query): |
|
145 | def _namespace_from_query(namespace, query): | |
146 | # cache namespace - the token handed in by the |
|
146 | # cache namespace - the token handed in by the | |
147 | # option + class we're querying against |
|
147 | # option + class we're querying against | |
148 | namespace = " ".join([namespace] + [str(x) for x in query._entities]) |
|
148 | namespace = " ".join([namespace] + [str(x) for x in query._entities]) | |
149 |
|
149 | |||
150 | # memcached wants this |
|
150 | # memcached wants this | |
151 | namespace = namespace.replace(' ', '_') |
|
151 | namespace = namespace.replace(' ', '_') | |
152 |
|
152 | |||
153 | return namespace |
|
153 | return namespace | |
154 |
|
154 | |||
155 | def _set_cache_parameters(query, region, namespace, cache_key): |
|
155 | def _set_cache_parameters(query, region, namespace, cache_key): | |
156 |
|
156 | |||
157 | if hasattr(query, '_cache_parameters'): |
|
157 | if hasattr(query, '_cache_parameters'): | |
158 | region, namespace, cache_key = query._cache_parameters |
|
158 | region, namespace, cache_key = query._cache_parameters | |
159 | raise ValueError("This query is already configured " |
|
159 | raise ValueError("This query is already configured " | |
160 | "for region %r namespace %r" % |
|
160 | "for region %r namespace %r" % | |
161 | (region, namespace) |
|
161 | (region, namespace) | |
162 | ) |
|
162 | ) | |
163 | query._cache_parameters = region, namespace, cache_key |
|
163 | query._cache_parameters = region, namespace, cache_key | |
164 |
|
164 | |||
165 | class FromCache(MapperOption): |
|
165 | class FromCache(MapperOption): | |
166 | """Specifies that a Query should load results from a cache.""" |
|
166 | """Specifies that a Query should load results from a cache.""" | |
167 |
|
167 | |||
168 | propagate_to_loaders = False |
|
168 | propagate_to_loaders = False | |
169 |
|
169 | |||
170 | def __init__(self, region, namespace, cache_key=None): |
|
170 | def __init__(self, region, namespace, cache_key=None): | |
171 | """Construct a new FromCache. |
|
171 | """Construct a new FromCache. | |
172 |
|
172 | |||
173 | :param region: the cache region. Should be a |
|
173 | :param region: the cache region. Should be a | |
174 | region configured in the Beaker CacheManager. |
|
174 | region configured in the Beaker CacheManager. | |
175 |
|
175 | |||
176 | :param namespace: the cache namespace. Should |
|
176 | :param namespace: the cache namespace. Should | |
177 | be a name uniquely describing the target Query's |
|
177 | be a name uniquely describing the target Query's | |
178 | lexical structure. |
|
178 | lexical structure. | |
179 |
|
179 | |||
180 | :param cache_key: optional. A string cache key |
|
180 | :param cache_key: optional. A string cache key | |
181 | that will serve as the key to the query. Use this |
|
181 | that will serve as the key to the query. Use this | |
182 | if your query has a huge amount of parameters (such |
|
182 | if your query has a huge amount of parameters (such | |
183 | as when using in_()) which correspond more simply to |
|
183 | as when using in_()) which correspond more simply to | |
184 | some other identifier. |
|
184 | some other identifier. | |
185 |
|
185 | |||
186 | """ |
|
186 | """ | |
187 | self.region = region |
|
187 | self.region = region | |
188 | self.namespace = namespace |
|
188 | self.namespace = namespace | |
189 | self.cache_key = cache_key |
|
189 | self.cache_key = cache_key | |
190 |
|
190 | |||
191 | def process_query(self, query): |
|
191 | def process_query(self, query): | |
192 | """Process a Query during normal loading operation.""" |
|
192 | """Process a Query during normal loading operation.""" | |
193 |
|
193 | |||
194 | _set_cache_parameters(query, self.region, self.namespace, self.cache_key) |
|
194 | _set_cache_parameters(query, self.region, self.namespace, self.cache_key) | |
195 |
|
195 | |||
196 | class RelationshipCache(MapperOption): |
|
196 | class RelationshipCache(MapperOption): | |
197 | """Specifies that a Query as called within a "lazy load" |
|
197 | """Specifies that a Query as called within a "lazy load" | |
198 | should load results from a cache.""" |
|
198 | should load results from a cache.""" | |
199 |
|
199 | |||
200 | propagate_to_loaders = True |
|
200 | propagate_to_loaders = True | |
201 |
|
201 | |||
202 | def __init__(self, region, namespace, attribute): |
|
202 | def __init__(self, region, namespace, attribute): | |
203 | """Construct a new RelationshipCache. |
|
203 | """Construct a new RelationshipCache. | |
204 |
|
204 | |||
205 | :param region: the cache region. Should be a |
|
205 | :param region: the cache region. Should be a | |
206 | region configured in the Beaker CacheManager. |
|
206 | region configured in the Beaker CacheManager. | |
207 |
|
207 | |||
208 | :param namespace: the cache namespace. Should |
|
208 | :param namespace: the cache namespace. Should | |
209 | be a name uniquely describing the target Query's |
|
209 | be a name uniquely describing the target Query's | |
210 | lexical structure. |
|
210 | lexical structure. | |
211 |
|
211 | |||
212 | :param attribute: A Class.attribute which |
|
212 | :param attribute: A Class.attribute which | |
213 | indicates a particular class relationship() whose |
|
213 | indicates a particular class relationship() whose | |
214 | lazy loader should be pulled from the cache. |
|
214 | lazy loader should be pulled from the cache. | |
215 |
|
215 | |||
216 | """ |
|
216 | """ | |
217 | self.region = region |
|
217 | self.region = region | |
218 | self.namespace = namespace |
|
218 | self.namespace = namespace | |
219 | self._relationship_options = { |
|
219 | self._relationship_options = { | |
220 | (attribute.property.parent.class_, attribute.property.key) : self |
|
220 | (attribute.property.parent.class_, attribute.property.key) : self | |
221 | } |
|
221 | } | |
222 |
|
222 | |||
223 | def process_query_conditionally(self, query): |
|
223 | def process_query_conditionally(self, query): | |
224 | """Process a Query that is used within a lazy loader. |
|
224 | """Process a Query that is used within a lazy loader. | |
225 |
|
225 | |||
226 | (the process_query_conditionally() method is a SQLAlchemy |
|
226 | (the process_query_conditionally() method is a SQLAlchemy | |
227 | hook invoked only within lazyload.) |
|
227 | hook invoked only within lazyload.) | |
228 |
|
228 | |||
229 | """ |
|
229 | """ | |
230 | if query._current_path: |
|
230 | if query._current_path: | |
231 | mapper, key = query._current_path[-2:] |
|
231 | mapper, key = query._current_path[-2:] | |
232 |
|
232 | |||
233 | for cls in mapper.class_.__mro__: |
|
233 | for cls in mapper.class_.__mro__: | |
234 | if (cls, key) in self._relationship_options: |
|
234 | if (cls, key) in self._relationship_options: | |
235 | relationship_option = self._relationship_options[(cls, key)] |
|
235 | relationship_option = self._relationship_options[(cls, key)] | |
236 | _set_cache_parameters( |
|
236 | _set_cache_parameters( | |
237 | query, |
|
237 | query, | |
238 | relationship_option.region, |
|
238 | relationship_option.region, | |
239 | relationship_option.namespace, |
|
239 | relationship_option.namespace, | |
240 | None) |
|
240 | None) | |
241 |
|
241 | |||
242 | def and_(self, option): |
|
242 | def and_(self, option): | |
243 | """Chain another RelationshipCache option to this one. |
|
243 | """Chain another RelationshipCache option to this one. | |
244 |
|
244 | |||
245 | While many RelationshipCache objects can be specified on a single |
|
245 | While many RelationshipCache objects can be specified on a single | |
246 | Query separately, chaining them together allows for a more efficient |
|
246 | Query separately, chaining them together allows for a more efficient | |
247 | lookup during load. |
|
247 | lookup during load. | |
248 |
|
248 | |||
249 | """ |
|
249 | """ | |
250 | self._relationship_options.update(option._relationship_options) |
|
250 | self._relationship_options.update(option._relationship_options) | |
251 | return self |
|
251 | return self | |
252 |
|
252 | |||
253 |
|
253 | |||
254 | def _params_from_query(query): |
|
254 | def _params_from_query(query): | |
255 | """Pull the bind parameter values from a query. |
|
255 | """Pull the bind parameter values from a query. | |
256 |
|
256 | |||
257 | This takes into account any scalar attribute bindparam set up. |
|
257 | This takes into account any scalar attribute bindparam set up. | |
258 |
|
258 | |||
259 | E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) |
|
259 | E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) | |
260 | would return [5, 7]. |
|
260 | would return [5, 7]. | |
261 |
|
261 | |||
262 | """ |
|
262 | """ | |
263 | v = [] |
|
263 | v = [] | |
264 | def visit_bindparam(bind): |
|
264 | def visit_bindparam(bind): | |
265 | value = query._params.get(bind.key, bind.value) |
|
265 | value = query._params.get(bind.key, bind.value) | |
266 |
|
266 | |||
267 | # lazyloader may dig a callable in here, intended |
|
267 | # lazyloader may dig a callable in here, intended | |
268 | # to late-evaluate params after autoflush is called. |
|
268 | # to late-evaluate params after autoflush is called. | |
269 | # convert to a scalar value. |
|
269 | # convert to a scalar value. | |
270 | if callable(value): |
|
270 | if callable(value): | |
271 | value = value() |
|
271 | value = value() | |
272 |
|
272 | |||
273 | v.append(value) |
|
273 | v.append(value) | |
274 | if query._criterion is not None: |
|
274 | if query._criterion is not None: | |
275 | visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) |
|
275 | visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) | |
276 | return v |
|
276 | return v |
General Comments 0
You need to be logged in to leave comments.
Login now