##// END OF EJS Templates
further fix/tests for record edits clobbering DictDB
MinRK -
Show More
@@ -1,216 +1,216 b''
1 """A Task logger that presents our DB interface,
1 """A Task logger that presents our DB interface,
2 but exists entirely in memory and implemented with dicts.
2 but exists entirely in memory and implemented with dicts.
3
3
4 Authors:
4 Authors:
5
5
6 * Min RK
6 * Min RK
7
7
8
8
9 TaskRecords are dicts of the form:
9 TaskRecords are dicts of the form:
10 {
10 {
11 'msg_id' : str(uuid),
11 'msg_id' : str(uuid),
12 'client_uuid' : str(uuid),
12 'client_uuid' : str(uuid),
13 'engine_uuid' : str(uuid) or None,
13 'engine_uuid' : str(uuid) or None,
14 'header' : dict(header),
14 'header' : dict(header),
15 'content': dict(content),
15 'content': dict(content),
16 'buffers': list(buffers),
16 'buffers': list(buffers),
17 'submitted': datetime,
17 'submitted': datetime,
18 'started': datetime or None,
18 'started': datetime or None,
19 'completed': datetime or None,
19 'completed': datetime or None,
20 'resubmitted': datetime or None,
20 'resubmitted': datetime or None,
21 'result_header' : dict(header) or None,
21 'result_header' : dict(header) or None,
22 'result_content' : dict(content) or None,
22 'result_content' : dict(content) or None,
23 'result_buffers' : list(buffers) or None,
23 'result_buffers' : list(buffers) or None,
24 }
24 }
25 With this info, many of the special categories of tasks can be defined by query:
25 With this info, many of the special categories of tasks can be defined by query:
26
26
27 pending: completed is None
27 pending: completed is None
28 client's outstanding: client_uuid = uuid && completed is None
28 client's outstanding: client_uuid = uuid && completed is None
29 MIA: arrived is None (and completed is None)
29 MIA: arrived is None (and completed is None)
30 etc.
30 etc.
31
31
32 EngineRecords are dicts of the form:
32 EngineRecords are dicts of the form:
33 {
33 {
34 'eid' : int(id),
34 'eid' : int(id),
35 'uuid': str(uuid)
35 'uuid': str(uuid)
36 }
36 }
37 This may be extended, but is currently.
37 This may be extended, but is currently.
38
38
39 We support a subset of mongodb operators:
39 We support a subset of mongodb operators:
40 $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
40 $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
41 """
41 """
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # Copyright (C) 2010-2011 The IPython Development Team
43 # Copyright (C) 2010-2011 The IPython Development Team
44 #
44 #
45 # Distributed under the terms of the BSD License. The full license is in
45 # Distributed under the terms of the BSD License. The full license is in
46 # the file COPYING, distributed as part of this software.
46 # the file COPYING, distributed as part of this software.
47 #-----------------------------------------------------------------------------
47 #-----------------------------------------------------------------------------
48
48
49 from copy import copy
49 from copy import deepcopy as copy
50 from datetime import datetime
50 from datetime import datetime
51
51
52 from IPython.config.configurable import LoggingConfigurable
52 from IPython.config.configurable import LoggingConfigurable
53
53
54 from IPython.utils.traitlets import Dict, Unicode, Instance
54 from IPython.utils.traitlets import Dict, Unicode, Instance
55
55
56 filters = {
56 filters = {
57 '$lt' : lambda a,b: a < b,
57 '$lt' : lambda a,b: a < b,
58 '$gt' : lambda a,b: b > a,
58 '$gt' : lambda a,b: b > a,
59 '$eq' : lambda a,b: a == b,
59 '$eq' : lambda a,b: a == b,
60 '$ne' : lambda a,b: a != b,
60 '$ne' : lambda a,b: a != b,
61 '$lte': lambda a,b: a <= b,
61 '$lte': lambda a,b: a <= b,
62 '$gte': lambda a,b: a >= b,
62 '$gte': lambda a,b: a >= b,
63 '$in' : lambda a,b: a in b,
63 '$in' : lambda a,b: a in b,
64 '$nin': lambda a,b: a not in b,
64 '$nin': lambda a,b: a not in b,
65 '$all': lambda a,b: all([ a in bb for bb in b ]),
65 '$all': lambda a,b: all([ a in bb for bb in b ]),
66 '$mod': lambda a,b: a%b[0] == b[1],
66 '$mod': lambda a,b: a%b[0] == b[1],
67 '$exists' : lambda a,b: (b and a is not None) or (a is None and not b)
67 '$exists' : lambda a,b: (b and a is not None) or (a is None and not b)
68 }
68 }
69
69
70
70
71 class CompositeFilter(object):
71 class CompositeFilter(object):
72 """Composite filter for matching multiple properties."""
72 """Composite filter for matching multiple properties."""
73
73
74 def __init__(self, dikt):
74 def __init__(self, dikt):
75 self.tests = []
75 self.tests = []
76 self.values = []
76 self.values = []
77 for key, value in dikt.iteritems():
77 for key, value in dikt.iteritems():
78 self.tests.append(filters[key])
78 self.tests.append(filters[key])
79 self.values.append(value)
79 self.values.append(value)
80
80
81 def __call__(self, value):
81 def __call__(self, value):
82 for test,check in zip(self.tests, self.values):
82 for test,check in zip(self.tests, self.values):
83 if not test(value, check):
83 if not test(value, check):
84 return False
84 return False
85 return True
85 return True
86
86
87 class BaseDB(LoggingConfigurable):
87 class BaseDB(LoggingConfigurable):
88 """Empty Parent class so traitlets work on DB."""
88 """Empty Parent class so traitlets work on DB."""
89 # base configurable traits:
89 # base configurable traits:
90 session = Unicode("")
90 session = Unicode("")
91
91
92 class DictDB(BaseDB):
92 class DictDB(BaseDB):
93 """Basic in-memory dict-based object for saving Task Records.
93 """Basic in-memory dict-based object for saving Task Records.
94
94
95 This is the first object to present the DB interface
95 This is the first object to present the DB interface
96 for logging tasks out of memory.
96 for logging tasks out of memory.
97
97
98 The interface is based on MongoDB, so adding a MongoDB
98 The interface is based on MongoDB, so adding a MongoDB
99 backend should be straightforward.
99 backend should be straightforward.
100 """
100 """
101
101
102 _records = Dict()
102 _records = Dict()
103
103
104 def _match_one(self, rec, tests):
104 def _match_one(self, rec, tests):
105 """Check if a specific record matches tests."""
105 """Check if a specific record matches tests."""
106 for key,test in tests.iteritems():
106 for key,test in tests.iteritems():
107 if not test(rec.get(key, None)):
107 if not test(rec.get(key, None)):
108 return False
108 return False
109 return True
109 return True
110
110
111 def _match(self, check):
111 def _match(self, check):
112 """Find all the matches for a check dict."""
112 """Find all the matches for a check dict."""
113 matches = []
113 matches = []
114 tests = {}
114 tests = {}
115 for k,v in check.iteritems():
115 for k,v in check.iteritems():
116 if isinstance(v, dict):
116 if isinstance(v, dict):
117 tests[k] = CompositeFilter(v)
117 tests[k] = CompositeFilter(v)
118 else:
118 else:
119 tests[k] = lambda o: o==v
119 tests[k] = lambda o: o==v
120
120
121 for rec in self._records.itervalues():
121 for rec in self._records.itervalues():
122 if self._match_one(rec, tests):
122 if self._match_one(rec, tests):
123 matches.append(copy(rec))
123 matches.append(copy(rec))
124 return matches
124 return matches
125
125
126 def _extract_subdict(self, rec, keys):
126 def _extract_subdict(self, rec, keys):
127 """extract subdict of keys"""
127 """extract subdict of keys"""
128 d = {}
128 d = {}
129 d['msg_id'] = rec['msg_id']
129 d['msg_id'] = rec['msg_id']
130 for key in keys:
130 for key in keys:
131 d[key] = rec[key]
131 d[key] = rec[key]
132 return d
132 return copy(d)
133
133
134 def add_record(self, msg_id, rec):
134 def add_record(self, msg_id, rec):
135 """Add a new Task Record, by msg_id."""
135 """Add a new Task Record, by msg_id."""
136 if self._records.has_key(msg_id):
136 if self._records.has_key(msg_id):
137 raise KeyError("Already have msg_id %r"%(msg_id))
137 raise KeyError("Already have msg_id %r"%(msg_id))
138 self._records[msg_id] = rec
138 self._records[msg_id] = rec
139
139
140 def get_record(self, msg_id):
140 def get_record(self, msg_id):
141 """Get a specific Task Record, by msg_id."""
141 """Get a specific Task Record, by msg_id."""
142 if not msg_id in self._records:
142 if not msg_id in self._records:
143 raise KeyError("No such msg_id %r"%(msg_id))
143 raise KeyError("No such msg_id %r"%(msg_id))
144 return copy(self._records[msg_id])
144 return copy(self._records[msg_id])
145
145
146 def update_record(self, msg_id, rec):
146 def update_record(self, msg_id, rec):
147 """Update the data in an existing record."""
147 """Update the data in an existing record."""
148 self._records[msg_id].update(rec)
148 self._records[msg_id].update(rec)
149
149
150 def drop_matching_records(self, check):
150 def drop_matching_records(self, check):
151 """Remove a record from the DB."""
151 """Remove a record from the DB."""
152 matches = self._match(check)
152 matches = self._match(check)
153 for m in matches:
153 for m in matches:
154 del self._records[m['msg_id']]
154 del self._records[m['msg_id']]
155
155
156 def drop_record(self, msg_id):
156 def drop_record(self, msg_id):
157 """Remove a record from the DB."""
157 """Remove a record from the DB."""
158 del self._records[msg_id]
158 del self._records[msg_id]
159
159
160
160
161 def find_records(self, check, keys=None):
161 def find_records(self, check, keys=None):
162 """Find records matching a query dict, optionally extracting subset of keys.
162 """Find records matching a query dict, optionally extracting subset of keys.
163
163
164 Returns dict keyed by msg_id of matching records.
164 Returns dict keyed by msg_id of matching records.
165
165
166 Parameters
166 Parameters
167 ----------
167 ----------
168
168
169 check: dict
169 check: dict
170 mongodb-style query argument
170 mongodb-style query argument
171 keys: list of strs [optional]
171 keys: list of strs [optional]
172 if specified, the subset of keys to extract. msg_id will *always* be
172 if specified, the subset of keys to extract. msg_id will *always* be
173 included.
173 included.
174 """
174 """
175 matches = self._match(check)
175 matches = self._match(check)
176 if keys:
176 if keys:
177 return [ self._extract_subdict(rec, keys) for rec in matches ]
177 return [ self._extract_subdict(rec, keys) for rec in matches ]
178 else:
178 else:
179 return matches
179 return matches
180
180
181
181
182 def get_history(self):
182 def get_history(self):
183 """get all msg_ids, ordered by time submitted."""
183 """get all msg_ids, ordered by time submitted."""
184 msg_ids = self._records.keys()
184 msg_ids = self._records.keys()
185 return sorted(msg_ids, key=lambda m: self._records[m]['submitted'])
185 return sorted(msg_ids, key=lambda m: self._records[m]['submitted'])
186
186
187 class NoDB(DictDB):
187 class NoDB(DictDB):
188 """A blackhole db backend that actually stores no information.
188 """A blackhole db backend that actually stores no information.
189
189
190 Provides the full DB interface, but raises KeyErrors on any
190 Provides the full DB interface, but raises KeyErrors on any
191 method that tries to access the records. This can be used to
191 method that tries to access the records. This can be used to
192 minimize the memory footprint of the Hub when its record-keeping
192 minimize the memory footprint of the Hub when its record-keeping
193 functionality is not required.
193 functionality is not required.
194 """
194 """
195
195
196 def add_record(self, msg_id, record):
196 def add_record(self, msg_id, record):
197 pass
197 pass
198
198
199 def get_record(self, msg_id):
199 def get_record(self, msg_id):
200 raise KeyError("NoDB does not support record access")
200 raise KeyError("NoDB does not support record access")
201
201
202 def update_record(self, msg_id, record):
202 def update_record(self, msg_id, record):
203 pass
203 pass
204
204
205 def drop_matching_records(self, check):
205 def drop_matching_records(self, check):
206 pass
206 pass
207
207
208 def drop_record(self, msg_id):
208 def drop_record(self, msg_id):
209 pass
209 pass
210
210
211 def find_records(self, check, keys=None):
211 def find_records(self, check, keys=None):
212 raise KeyError("NoDB does not store information")
212 raise KeyError("NoDB does not store information")
213
213
214 def get_history(self):
214 def get_history(self):
215 raise KeyError("NoDB does not store information")
215 raise KeyError("NoDB does not store information")
216
216
@@ -1,243 +1,249 b''
1 """Tests for db backends
1 """Tests for db backends
2
2
3 Authors:
3 Authors:
4
4
5 * Min RK
5 * Min RK
6 """
6 """
7
7
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9 # Copyright (C) 2011 The IPython Development Team
9 # Copyright (C) 2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-------------------------------------------------------------------------------
13 #-------------------------------------------------------------------------------
14
14
15 #-------------------------------------------------------------------------------
15 #-------------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-------------------------------------------------------------------------------
17 #-------------------------------------------------------------------------------
18
18
19 from __future__ import division
19 from __future__ import division
20
20
21 import logging
21 import logging
22 import os
22 import os
23 import tempfile
23 import tempfile
24 import time
24 import time
25
25
26 from datetime import datetime, timedelta
26 from datetime import datetime, timedelta
27 from unittest import TestCase
27 from unittest import TestCase
28
28
29 from IPython.parallel import error
29 from IPython.parallel import error
30 from IPython.parallel.controller.dictdb import DictDB
30 from IPython.parallel.controller.dictdb import DictDB
31 from IPython.parallel.controller.sqlitedb import SQLiteDB
31 from IPython.parallel.controller.sqlitedb import SQLiteDB
32 from IPython.parallel.controller.hub import init_record, empty_record
32 from IPython.parallel.controller.hub import init_record, empty_record
33
33
34 from IPython.testing import decorators as dec
34 from IPython.testing import decorators as dec
35 from IPython.zmq.session import Session
35 from IPython.zmq.session import Session
36
36
37
37
38 #-------------------------------------------------------------------------------
38 #-------------------------------------------------------------------------------
39 # TestCases
39 # TestCases
40 #-------------------------------------------------------------------------------
40 #-------------------------------------------------------------------------------
41
41
42
42
43 def setup():
43 def setup():
44 global temp_db
44 global temp_db
45 temp_db = tempfile.NamedTemporaryFile(suffix='.db').name
45 temp_db = tempfile.NamedTemporaryFile(suffix='.db').name
46
46
47
47
48 class TestDictBackend(TestCase):
48 class TestDictBackend(TestCase):
49 def setUp(self):
49 def setUp(self):
50 self.session = Session()
50 self.session = Session()
51 self.db = self.create_db()
51 self.db = self.create_db()
52 self.load_records(16)
52 self.load_records(16)
53
53
54 def create_db(self):
54 def create_db(self):
55 return DictDB()
55 return DictDB()
56
56
57 def load_records(self, n=1):
57 def load_records(self, n=1):
58 """load n records for testing"""
58 """load n records for testing"""
59 #sleep 1/10 s, to ensure timestamp is different to previous calls
59 #sleep 1/10 s, to ensure timestamp is different to previous calls
60 time.sleep(0.1)
60 time.sleep(0.1)
61 msg_ids = []
61 msg_ids = []
62 for i in range(n):
62 for i in range(n):
63 msg = self.session.msg('apply_request', content=dict(a=5))
63 msg = self.session.msg('apply_request', content=dict(a=5))
64 msg['buffers'] = []
64 msg['buffers'] = []
65 rec = init_record(msg)
65 rec = init_record(msg)
66 msg_id = msg['header']['msg_id']
66 msg_id = msg['header']['msg_id']
67 msg_ids.append(msg_id)
67 msg_ids.append(msg_id)
68 self.db.add_record(msg_id, rec)
68 self.db.add_record(msg_id, rec)
69 return msg_ids
69 return msg_ids
70
70
71 def test_add_record(self):
71 def test_add_record(self):
72 before = self.db.get_history()
72 before = self.db.get_history()
73 self.load_records(5)
73 self.load_records(5)
74 after = self.db.get_history()
74 after = self.db.get_history()
75 self.assertEquals(len(after), len(before)+5)
75 self.assertEquals(len(after), len(before)+5)
76 self.assertEquals(after[:-5],before)
76 self.assertEquals(after[:-5],before)
77
77
78 def test_drop_record(self):
78 def test_drop_record(self):
79 msg_id = self.load_records()[-1]
79 msg_id = self.load_records()[-1]
80 rec = self.db.get_record(msg_id)
80 rec = self.db.get_record(msg_id)
81 self.db.drop_record(msg_id)
81 self.db.drop_record(msg_id)
82 self.assertRaises(KeyError,self.db.get_record, msg_id)
82 self.assertRaises(KeyError,self.db.get_record, msg_id)
83
83
84 def _round_to_millisecond(self, dt):
84 def _round_to_millisecond(self, dt):
85 """necessary because mongodb rounds microseconds"""
85 """necessary because mongodb rounds microseconds"""
86 micro = dt.microsecond
86 micro = dt.microsecond
87 extra = int(str(micro)[-3:])
87 extra = int(str(micro)[-3:])
88 return dt - timedelta(microseconds=extra)
88 return dt - timedelta(microseconds=extra)
89
89
90 def test_update_record(self):
90 def test_update_record(self):
91 now = self._round_to_millisecond(datetime.now())
91 now = self._round_to_millisecond(datetime.now())
92 #
92 #
93 msg_id = self.db.get_history()[-1]
93 msg_id = self.db.get_history()[-1]
94 rec1 = self.db.get_record(msg_id)
94 rec1 = self.db.get_record(msg_id)
95 data = {'stdout': 'hello there', 'completed' : now}
95 data = {'stdout': 'hello there', 'completed' : now}
96 self.db.update_record(msg_id, data)
96 self.db.update_record(msg_id, data)
97 rec2 = self.db.get_record(msg_id)
97 rec2 = self.db.get_record(msg_id)
98 self.assertEquals(rec2['stdout'], 'hello there')
98 self.assertEquals(rec2['stdout'], 'hello there')
99 self.assertEquals(rec2['completed'], now)
99 self.assertEquals(rec2['completed'], now)
100 rec1.update(data)
100 rec1.update(data)
101 self.assertEquals(rec1, rec2)
101 self.assertEquals(rec1, rec2)
102
102
103 # def test_update_record_bad(self):
103 # def test_update_record_bad(self):
104 # """test updating nonexistant records"""
104 # """test updating nonexistant records"""
105 # msg_id = str(uuid.uuid4())
105 # msg_id = str(uuid.uuid4())
106 # data = {'stdout': 'hello there'}
106 # data = {'stdout': 'hello there'}
107 # self.assertRaises(KeyError, self.db.update_record, msg_id, data)
107 # self.assertRaises(KeyError, self.db.update_record, msg_id, data)
108
108
109 def test_find_records_dt(self):
109 def test_find_records_dt(self):
110 """test finding records by date"""
110 """test finding records by date"""
111 hist = self.db.get_history()
111 hist = self.db.get_history()
112 middle = self.db.get_record(hist[len(hist)//2])
112 middle = self.db.get_record(hist[len(hist)//2])
113 tic = middle['submitted']
113 tic = middle['submitted']
114 before = self.db.find_records({'submitted' : {'$lt' : tic}})
114 before = self.db.find_records({'submitted' : {'$lt' : tic}})
115 after = self.db.find_records({'submitted' : {'$gte' : tic}})
115 after = self.db.find_records({'submitted' : {'$gte' : tic}})
116 self.assertEquals(len(before)+len(after),len(hist))
116 self.assertEquals(len(before)+len(after),len(hist))
117 for b in before:
117 for b in before:
118 self.assertTrue(b['submitted'] < tic)
118 self.assertTrue(b['submitted'] < tic)
119 for a in after:
119 for a in after:
120 self.assertTrue(a['submitted'] >= tic)
120 self.assertTrue(a['submitted'] >= tic)
121 same = self.db.find_records({'submitted' : tic})
121 same = self.db.find_records({'submitted' : tic})
122 for s in same:
122 for s in same:
123 self.assertTrue(s['submitted'] == tic)
123 self.assertTrue(s['submitted'] == tic)
124
124
125 def test_find_records_keys(self):
125 def test_find_records_keys(self):
126 """test extracting subset of record keys"""
126 """test extracting subset of record keys"""
127 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
127 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
128 for rec in found:
128 for rec in found:
129 self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
129 self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
130
130
131 def test_find_records_msg_id(self):
131 def test_find_records_msg_id(self):
132 """ensure msg_id is always in found records"""
132 """ensure msg_id is always in found records"""
133 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
133 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
134 for rec in found:
134 for rec in found:
135 self.assertTrue('msg_id' in rec.keys())
135 self.assertTrue('msg_id' in rec.keys())
136 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted'])
136 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted'])
137 for rec in found:
137 for rec in found:
138 self.assertTrue('msg_id' in rec.keys())
138 self.assertTrue('msg_id' in rec.keys())
139 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['msg_id'])
139 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['msg_id'])
140 for rec in found:
140 for rec in found:
141 self.assertTrue('msg_id' in rec.keys())
141 self.assertTrue('msg_id' in rec.keys())
142
142
143 def test_find_records_in(self):
143 def test_find_records_in(self):
144 """test finding records with '$in','$nin' operators"""
144 """test finding records with '$in','$nin' operators"""
145 hist = self.db.get_history()
145 hist = self.db.get_history()
146 even = hist[::2]
146 even = hist[::2]
147 odd = hist[1::2]
147 odd = hist[1::2]
148 recs = self.db.find_records({ 'msg_id' : {'$in' : even}})
148 recs = self.db.find_records({ 'msg_id' : {'$in' : even}})
149 found = [ r['msg_id'] for r in recs ]
149 found = [ r['msg_id'] for r in recs ]
150 self.assertEquals(set(even), set(found))
150 self.assertEquals(set(even), set(found))
151 recs = self.db.find_records({ 'msg_id' : {'$nin' : even}})
151 recs = self.db.find_records({ 'msg_id' : {'$nin' : even}})
152 found = [ r['msg_id'] for r in recs ]
152 found = [ r['msg_id'] for r in recs ]
153 self.assertEquals(set(odd), set(found))
153 self.assertEquals(set(odd), set(found))
154
154
155 def test_get_history(self):
155 def test_get_history(self):
156 msg_ids = self.db.get_history()
156 msg_ids = self.db.get_history()
157 latest = datetime(1984,1,1)
157 latest = datetime(1984,1,1)
158 for msg_id in msg_ids:
158 for msg_id in msg_ids:
159 rec = self.db.get_record(msg_id)
159 rec = self.db.get_record(msg_id)
160 newt = rec['submitted']
160 newt = rec['submitted']
161 self.assertTrue(newt >= latest)
161 self.assertTrue(newt >= latest)
162 latest = newt
162 latest = newt
163 msg_id = self.load_records(1)[-1]
163 msg_id = self.load_records(1)[-1]
164 self.assertEquals(self.db.get_history()[-1],msg_id)
164 self.assertEquals(self.db.get_history()[-1],msg_id)
165
165
166 def test_datetime(self):
166 def test_datetime(self):
167 """get/set timestamps with datetime objects"""
167 """get/set timestamps with datetime objects"""
168 msg_id = self.db.get_history()[-1]
168 msg_id = self.db.get_history()[-1]
169 rec = self.db.get_record(msg_id)
169 rec = self.db.get_record(msg_id)
170 self.assertTrue(isinstance(rec['submitted'], datetime))
170 self.assertTrue(isinstance(rec['submitted'], datetime))
171 self.db.update_record(msg_id, dict(completed=datetime.now()))
171 self.db.update_record(msg_id, dict(completed=datetime.now()))
172 rec = self.db.get_record(msg_id)
172 rec = self.db.get_record(msg_id)
173 self.assertTrue(isinstance(rec['completed'], datetime))
173 self.assertTrue(isinstance(rec['completed'], datetime))
174
174
175 def test_drop_matching(self):
175 def test_drop_matching(self):
176 msg_ids = self.load_records(10)
176 msg_ids = self.load_records(10)
177 query = {'msg_id' : {'$in':msg_ids}}
177 query = {'msg_id' : {'$in':msg_ids}}
178 self.db.drop_matching_records(query)
178 self.db.drop_matching_records(query)
179 recs = self.db.find_records(query)
179 recs = self.db.find_records(query)
180 self.assertEquals(len(recs), 0)
180 self.assertEquals(len(recs), 0)
181
181
182 def test_null(self):
182 def test_null(self):
183 """test None comparison queries"""
183 """test None comparison queries"""
184 msg_ids = self.load_records(10)
184 msg_ids = self.load_records(10)
185
185
186 query = {'msg_id' : None}
186 query = {'msg_id' : None}
187 recs = self.db.find_records(query)
187 recs = self.db.find_records(query)
188 self.assertEquals(len(recs), 0)
188 self.assertEquals(len(recs), 0)
189
189
190 query = {'msg_id' : {'$ne' : None}}
190 query = {'msg_id' : {'$ne' : None}}
191 recs = self.db.find_records(query)
191 recs = self.db.find_records(query)
192 self.assertTrue(len(recs) >= 10)
192 self.assertTrue(len(recs) >= 10)
193
193
194 def test_pop_safe_get(self):
194 def test_pop_safe_get(self):
195 """editing query results shouldn't affect record [get]"""
195 """editing query results shouldn't affect record [get]"""
196 msg_id = self.db.get_history()[-1]
196 msg_id = self.db.get_history()[-1]
197 rec = self.db.get_record(msg_id)
197 rec = self.db.get_record(msg_id)
198 rec.pop('buffers')
198 rec.pop('buffers')
199 rec['garbage'] = 'hello'
199 rec['garbage'] = 'hello'
200 rec['header']['msg_id'] = 'fubar'
200 rec2 = self.db.get_record(msg_id)
201 rec2 = self.db.get_record(msg_id)
201 self.assertTrue('buffers' in rec2)
202 self.assertTrue('buffers' in rec2)
202 self.assertFalse('garbage' in rec2)
203 self.assertFalse('garbage' in rec2)
204 self.assertEquals(rec2['header']['msg_id'], msg_id)
203
205
204 def test_pop_safe_find(self):
206 def test_pop_safe_find(self):
205 """editing query results shouldn't affect record [find]"""
207 """editing query results shouldn't affect record [find]"""
206 msg_id = self.db.get_history()[-1]
208 msg_id = self.db.get_history()[-1]
207 rec = self.db.find_records({'msg_id' : msg_id})[0]
209 rec = self.db.find_records({'msg_id' : msg_id})[0]
208 rec.pop('buffers')
210 rec.pop('buffers')
209 rec['garbage'] = 'hello'
211 rec['garbage'] = 'hello'
212 rec['header']['msg_id'] = 'fubar'
210 rec2 = self.db.find_records({'msg_id' : msg_id})[0]
213 rec2 = self.db.find_records({'msg_id' : msg_id})[0]
211 self.assertTrue('buffers' in rec2)
214 self.assertTrue('buffers' in rec2)
212 self.assertFalse('garbage' in rec2)
215 self.assertFalse('garbage' in rec2)
216 self.assertEquals(rec2['header']['msg_id'], msg_id)
213
217
214 def test_pop_safe_find_keys(self):
218 def test_pop_safe_find_keys(self):
215 """editing query results shouldn't affect record [find+keys]"""
219 """editing query results shouldn't affect record [find+keys]"""
216 msg_id = self.db.get_history()[-1]
220 msg_id = self.db.get_history()[-1]
217 rec = self.db.find_records({'msg_id' : msg_id}, keys=['buffers'])[0]
221 rec = self.db.find_records({'msg_id' : msg_id}, keys=['buffers', 'header'])[0]
218 rec.pop('buffers')
222 rec.pop('buffers')
219 rec['garbage'] = 'hello'
223 rec['garbage'] = 'hello'
224 rec['header']['msg_id'] = 'fubar'
220 rec2 = self.db.find_records({'msg_id' : msg_id})[0]
225 rec2 = self.db.find_records({'msg_id' : msg_id})[0]
221 self.assertTrue('buffers' in rec2)
226 self.assertTrue('buffers' in rec2)
222 self.assertFalse('garbage' in rec2)
227 self.assertFalse('garbage' in rec2)
228 self.assertEquals(rec2['header']['msg_id'], msg_id)
223
229
224
230
225 class TestSQLiteBackend(TestDictBackend):
231 class TestSQLiteBackend(TestDictBackend):
226
232
227 @dec.skip_without('sqlite3')
233 @dec.skip_without('sqlite3')
228 def create_db(self):
234 def create_db(self):
229 location, fname = os.path.split(temp_db)
235 location, fname = os.path.split(temp_db)
230 log = logging.getLogger('test')
236 log = logging.getLogger('test')
231 log.setLevel(logging.CRITICAL)
237 log.setLevel(logging.CRITICAL)
232 return SQLiteDB(location=location, fname=fname, log=log)
238 return SQLiteDB(location=location, fname=fname, log=log)
233
239
234 def tearDown(self):
240 def tearDown(self):
235 self.db._db.close()
241 self.db._db.close()
236
242
237
243
238 def teardown():
244 def teardown():
239 """cleanup task db file after all tests have run"""
245 """cleanup task db file after all tests have run"""
240 try:
246 try:
241 os.remove(temp_db)
247 os.remove(temp_db)
242 except:
248 except:
243 pass
249 pass
General Comments 0
You need to be logged in to leave comments. Login now