##// END OF EJS Templates
remove unused namedtuple import
MinRK -
Show More
@@ -1,328 +1,327 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """Pickle related utilities. Perhaps this should be called 'can'."""
3 """Pickle related utilities. Perhaps this should be called 'can'."""
4
4
5 __docformat__ = "restructuredtext en"
5 __docformat__ = "restructuredtext en"
6
6
7 #-------------------------------------------------------------------------------
7 #-------------------------------------------------------------------------------
8 # Copyright (C) 2008-2011 The IPython Development Team
8 # Copyright (C) 2008-2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-------------------------------------------------------------------------------
12 #-------------------------------------------------------------------------------
13
13
14 #-------------------------------------------------------------------------------
14 #-------------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-------------------------------------------------------------------------------
16 #-------------------------------------------------------------------------------
17
17
18 import copy
18 import copy
19 import logging
19 import logging
20 import sys
20 import sys
21 from collections import namedtuple
22 from types import FunctionType
21 from types import FunctionType
23
22
24 try:
23 try:
25 import cPickle as pickle
24 import cPickle as pickle
26 except ImportError:
25 except ImportError:
27 import pickle
26 import pickle
28
27
29 try:
28 try:
30 import numpy
29 import numpy
31 except:
30 except:
32 numpy = None
31 numpy = None
33
32
34 import codeutil
33 import codeutil
35 import py3compat
34 import py3compat
36 from importstring import import_item
35 from importstring import import_item
37
36
38 from IPython.config import Application
37 from IPython.config import Application
39
38
40 if py3compat.PY3:
39 if py3compat.PY3:
41 buffer = memoryview
40 buffer = memoryview
42 class_type = type
41 class_type = type
43 else:
42 else:
44 from types import ClassType
43 from types import ClassType
45 class_type = (type, ClassType)
44 class_type = (type, ClassType)
46
45
47 #-------------------------------------------------------------------------------
46 #-------------------------------------------------------------------------------
48 # Classes
47 # Classes
49 #-------------------------------------------------------------------------------
48 #-------------------------------------------------------------------------------
50
49
51
50
52 class CannedObject(object):
51 class CannedObject(object):
53 def __init__(self, obj, keys=[]):
52 def __init__(self, obj, keys=[]):
54 self.keys = keys
53 self.keys = keys
55 self.obj = copy.copy(obj)
54 self.obj = copy.copy(obj)
56 for key in keys:
55 for key in keys:
57 setattr(self.obj, key, can(getattr(obj, key)))
56 setattr(self.obj, key, can(getattr(obj, key)))
58
57
59 self.buffers = []
58 self.buffers = []
60
59
61 def get_object(self, g=None):
60 def get_object(self, g=None):
62 if g is None:
61 if g is None:
63 g = {}
62 g = {}
64 for key in self.keys:
63 for key in self.keys:
65 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
64 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
66 return self.obj
65 return self.obj
67
66
68
67
69 class Reference(CannedObject):
68 class Reference(CannedObject):
70 """object for wrapping a remote reference by name."""
69 """object for wrapping a remote reference by name."""
71 def __init__(self, name):
70 def __init__(self, name):
72 if not isinstance(name, basestring):
71 if not isinstance(name, basestring):
73 raise TypeError("illegal name: %r"%name)
72 raise TypeError("illegal name: %r"%name)
74 self.name = name
73 self.name = name
75 self.buffers = []
74 self.buffers = []
76
75
77 def __repr__(self):
76 def __repr__(self):
78 return "<Reference: %r>"%self.name
77 return "<Reference: %r>"%self.name
79
78
80 def get_object(self, g=None):
79 def get_object(self, g=None):
81 if g is None:
80 if g is None:
82 g = {}
81 g = {}
83
82
84 return eval(self.name, g)
83 return eval(self.name, g)
85
84
86
85
87 class CannedFunction(CannedObject):
86 class CannedFunction(CannedObject):
88
87
89 def __init__(self, f):
88 def __init__(self, f):
90 self._check_type(f)
89 self._check_type(f)
91 self.code = f.func_code
90 self.code = f.func_code
92 if f.func_defaults:
91 if f.func_defaults:
93 self.defaults = [ can(fd) for fd in f.func_defaults ]
92 self.defaults = [ can(fd) for fd in f.func_defaults ]
94 else:
93 else:
95 self.defaults = None
94 self.defaults = None
96 self.module = f.__module__ or '__main__'
95 self.module = f.__module__ or '__main__'
97 self.__name__ = f.__name__
96 self.__name__ = f.__name__
98 self.buffers = []
97 self.buffers = []
99
98
100 def _check_type(self, obj):
99 def _check_type(self, obj):
101 assert isinstance(obj, FunctionType), "Not a function type"
100 assert isinstance(obj, FunctionType), "Not a function type"
102
101
103 def get_object(self, g=None):
102 def get_object(self, g=None):
104 # try to load function back into its module:
103 # try to load function back into its module:
105 if not self.module.startswith('__'):
104 if not self.module.startswith('__'):
106 __import__(self.module)
105 __import__(self.module)
107 g = sys.modules[self.module].__dict__
106 g = sys.modules[self.module].__dict__
108
107
109 if g is None:
108 if g is None:
110 g = {}
109 g = {}
111 if self.defaults:
110 if self.defaults:
112 defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
111 defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
113 else:
112 else:
114 defaults = None
113 defaults = None
115 newFunc = FunctionType(self.code, g, self.__name__, defaults)
114 newFunc = FunctionType(self.code, g, self.__name__, defaults)
116 return newFunc
115 return newFunc
117
116
118 class CannedClass(CannedObject):
117 class CannedClass(CannedObject):
119
118
120 def __init__(self, cls):
119 def __init__(self, cls):
121 self._check_type(cls)
120 self._check_type(cls)
122 self.name = cls.__name__
121 self.name = cls.__name__
123 self.old_style = not isinstance(cls, type)
122 self.old_style = not isinstance(cls, type)
124 self._canned_dict = {}
123 self._canned_dict = {}
125 for k,v in cls.__dict__.items():
124 for k,v in cls.__dict__.items():
126 if k not in ('__weakref__', '__dict__'):
125 if k not in ('__weakref__', '__dict__'):
127 self._canned_dict[k] = can(v)
126 self._canned_dict[k] = can(v)
128 if self.old_style:
127 if self.old_style:
129 mro = []
128 mro = []
130 else:
129 else:
131 mro = cls.mro()
130 mro = cls.mro()
132
131
133 self.parents = [ can(c) for c in mro[1:] ]
132 self.parents = [ can(c) for c in mro[1:] ]
134 self.buffers = []
133 self.buffers = []
135
134
136 def _check_type(self, obj):
135 def _check_type(self, obj):
137 assert isinstance(obj, class_type), "Not a class type"
136 assert isinstance(obj, class_type), "Not a class type"
138
137
139 def get_object(self, g=None):
138 def get_object(self, g=None):
140 parents = tuple(uncan(p, g) for p in self.parents)
139 parents = tuple(uncan(p, g) for p in self.parents)
141 return type(self.name, parents, uncan_dict(self._canned_dict, g=g))
140 return type(self.name, parents, uncan_dict(self._canned_dict, g=g))
142
141
143 class CannedArray(CannedObject):
142 class CannedArray(CannedObject):
144 def __init__(self, obj):
143 def __init__(self, obj):
145 self.shape = obj.shape
144 self.shape = obj.shape
146 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
145 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
147 if sum(obj.shape) == 0:
146 if sum(obj.shape) == 0:
148 # just pickle it
147 # just pickle it
149 self.buffers = [pickle.dumps(obj, -1)]
148 self.buffers = [pickle.dumps(obj, -1)]
150 else:
149 else:
151 # ensure contiguous
150 # ensure contiguous
152 obj = numpy.ascontiguousarray(obj, dtype=None)
151 obj = numpy.ascontiguousarray(obj, dtype=None)
153 self.buffers = [buffer(obj)]
152 self.buffers = [buffer(obj)]
154
153
155 def get_object(self, g=None):
154 def get_object(self, g=None):
156 data = self.buffers[0]
155 data = self.buffers[0]
157 if sum(self.shape) == 0:
156 if sum(self.shape) == 0:
158 # no shape, we just pickled it
157 # no shape, we just pickled it
159 return pickle.loads(data)
158 return pickle.loads(data)
160 else:
159 else:
161 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
160 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
162
161
163
162
164 class CannedBytes(CannedObject):
163 class CannedBytes(CannedObject):
165 wrap = bytes
164 wrap = bytes
166 def __init__(self, obj):
165 def __init__(self, obj):
167 self.buffers = [obj]
166 self.buffers = [obj]
168
167
169 def get_object(self, g=None):
168 def get_object(self, g=None):
170 data = self.buffers[0]
169 data = self.buffers[0]
171 return self.wrap(data)
170 return self.wrap(data)
172
171
173 def CannedBuffer(CannedBytes):
172 def CannedBuffer(CannedBytes):
174 wrap = buffer
173 wrap = buffer
175
174
176 #-------------------------------------------------------------------------------
175 #-------------------------------------------------------------------------------
177 # Functions
176 # Functions
178 #-------------------------------------------------------------------------------
177 #-------------------------------------------------------------------------------
179
178
180 def _logger():
179 def _logger():
181 """get the logger for the current Application
180 """get the logger for the current Application
182
181
183 the root logger will be used if no Application is running
182 the root logger will be used if no Application is running
184 """
183 """
185 if Application.initialized():
184 if Application.initialized():
186 logger = Application.instance().log
185 logger = Application.instance().log
187 else:
186 else:
188 logger = logging.getLogger()
187 logger = logging.getLogger()
189 if not logger.handlers:
188 if not logger.handlers:
190 logging.basicConfig()
189 logging.basicConfig()
191
190
192 return logger
191 return logger
193
192
194 def _import_mapping(mapping, original=None):
193 def _import_mapping(mapping, original=None):
195 """import any string-keys in a type mapping
194 """import any string-keys in a type mapping
196
195
197 """
196 """
198 log = _logger()
197 log = _logger()
199 log.debug("Importing canning map")
198 log.debug("Importing canning map")
200 for key,value in mapping.items():
199 for key,value in mapping.items():
201 if isinstance(key, basestring):
200 if isinstance(key, basestring):
202 try:
201 try:
203 cls = import_item(key)
202 cls = import_item(key)
204 except Exception:
203 except Exception:
205 if original and key not in original:
204 if original and key not in original:
206 # only message on user-added classes
205 # only message on user-added classes
207 log.error("cannning class not importable: %r", key, exc_info=True)
206 log.error("cannning class not importable: %r", key, exc_info=True)
208 mapping.pop(key)
207 mapping.pop(key)
209 else:
208 else:
210 mapping[cls] = mapping.pop(key)
209 mapping[cls] = mapping.pop(key)
211
210
212 def istype(obj, check):
211 def istype(obj, check):
213 """like isinstance(obj, check), but strict
212 """like isinstance(obj, check), but strict
214
213
215 This won't catch subclasses.
214 This won't catch subclasses.
216 """
215 """
217 if isinstance(check, tuple):
216 if isinstance(check, tuple):
218 for cls in check:
217 for cls in check:
219 if type(obj) is cls:
218 if type(obj) is cls:
220 return True
219 return True
221 return False
220 return False
222 else:
221 else:
223 return type(obj) is check
222 return type(obj) is check
224
223
225 def can(obj):
224 def can(obj):
226 """prepare an object for pickling"""
225 """prepare an object for pickling"""
227
226
228 import_needed = False
227 import_needed = False
229
228
230 for cls,canner in can_map.iteritems():
229 for cls,canner in can_map.iteritems():
231 if isinstance(cls, basestring):
230 if isinstance(cls, basestring):
232 import_needed = True
231 import_needed = True
233 break
232 break
234 elif istype(obj, cls):
233 elif istype(obj, cls):
235 return canner(obj)
234 return canner(obj)
236
235
237 if import_needed:
236 if import_needed:
238 # perform can_map imports, then try again
237 # perform can_map imports, then try again
239 # this will usually only happen once
238 # this will usually only happen once
240 _import_mapping(can_map, _original_can_map)
239 _import_mapping(can_map, _original_can_map)
241 return can(obj)
240 return can(obj)
242
241
243 return obj
242 return obj
244
243
245 def can_class(obj):
244 def can_class(obj):
246 if isinstance(obj, class_type) and obj.__module__ == '__main__':
245 if isinstance(obj, class_type) and obj.__module__ == '__main__':
247 return CannedClass(obj)
246 return CannedClass(obj)
248 else:
247 else:
249 return obj
248 return obj
250
249
251 def can_dict(obj):
250 def can_dict(obj):
252 """can the *values* of a dict"""
251 """can the *values* of a dict"""
253 if istype(obj, dict):
252 if istype(obj, dict):
254 newobj = {}
253 newobj = {}
255 for k, v in obj.iteritems():
254 for k, v in obj.iteritems():
256 newobj[k] = can(v)
255 newobj[k] = can(v)
257 return newobj
256 return newobj
258 else:
257 else:
259 return obj
258 return obj
260
259
261 sequence_types = (list, tuple, set)
260 sequence_types = (list, tuple, set)
262
261
263 def can_sequence(obj):
262 def can_sequence(obj):
264 """can the elements of a sequence"""
263 """can the elements of a sequence"""
265 if istype(obj, sequence_types):
264 if istype(obj, sequence_types):
266 t = type(obj)
265 t = type(obj)
267 return t([can(i) for i in obj])
266 return t([can(i) for i in obj])
268 else:
267 else:
269 return obj
268 return obj
270
269
271 def uncan(obj, g=None):
270 def uncan(obj, g=None):
272 """invert canning"""
271 """invert canning"""
273
272
274 import_needed = False
273 import_needed = False
275 for cls,uncanner in uncan_map.iteritems():
274 for cls,uncanner in uncan_map.iteritems():
276 if isinstance(cls, basestring):
275 if isinstance(cls, basestring):
277 import_needed = True
276 import_needed = True
278 break
277 break
279 elif isinstance(obj, cls):
278 elif isinstance(obj, cls):
280 return uncanner(obj, g)
279 return uncanner(obj, g)
281
280
282 if import_needed:
281 if import_needed:
283 # perform uncan_map imports, then try again
282 # perform uncan_map imports, then try again
284 # this will usually only happen once
283 # this will usually only happen once
285 _import_mapping(uncan_map, _original_uncan_map)
284 _import_mapping(uncan_map, _original_uncan_map)
286 return uncan(obj, g)
285 return uncan(obj, g)
287
286
288 return obj
287 return obj
289
288
290 def uncan_dict(obj, g=None):
289 def uncan_dict(obj, g=None):
291 if istype(obj, dict):
290 if istype(obj, dict):
292 newobj = {}
291 newobj = {}
293 for k, v in obj.iteritems():
292 for k, v in obj.iteritems():
294 newobj[k] = uncan(v,g)
293 newobj[k] = uncan(v,g)
295 return newobj
294 return newobj
296 else:
295 else:
297 return obj
296 return obj
298
297
299 def uncan_sequence(obj, g=None):
298 def uncan_sequence(obj, g=None):
300 if istype(obj, sequence_types):
299 if istype(obj, sequence_types):
301 t = type(obj)
300 t = type(obj)
302 return t([uncan(i,g) for i in obj])
301 return t([uncan(i,g) for i in obj])
303 else:
302 else:
304 return obj
303 return obj
305
304
306
305
307 #-------------------------------------------------------------------------------
306 #-------------------------------------------------------------------------------
308 # API dictionaries
307 # API dictionaries
309 #-------------------------------------------------------------------------------
308 #-------------------------------------------------------------------------------
310
309
311 # These dicts can be extended for custom serialization of new objects
310 # These dicts can be extended for custom serialization of new objects
312
311
313 can_map = {
312 can_map = {
314 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
313 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
315 'numpy.ndarray' : CannedArray,
314 'numpy.ndarray' : CannedArray,
316 FunctionType : CannedFunction,
315 FunctionType : CannedFunction,
317 bytes : CannedBytes,
316 bytes : CannedBytes,
318 buffer : CannedBuffer,
317 buffer : CannedBuffer,
319 class_type : can_class,
318 class_type : can_class,
320 }
319 }
321
320
322 uncan_map = {
321 uncan_map = {
323 CannedObject : lambda obj, g: obj.get_object(g),
322 CannedObject : lambda obj, g: obj.get_object(g),
324 }
323 }
325
324
326 # for use in _import_mapping:
325 # for use in _import_mapping:
327 _original_can_map = can_map.copy()
326 _original_can_map = can_map.copy()
328 _original_uncan_map = uncan_map.copy()
327 _original_uncan_map = uncan_map.copy()
General Comments 0
You need to be logged in to leave comments. Login now