##// END OF EJS Templates
add istype to canning...
MinRK -
Show More
@@ -1,312 +1,325 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """Pickle related utilities. Perhaps this should be called 'can'."""
3 """Pickle related utilities. Perhaps this should be called 'can'."""
4
4
5 __docformat__ = "restructuredtext en"
5 __docformat__ = "restructuredtext en"
6
6
7 #-------------------------------------------------------------------------------
7 #-------------------------------------------------------------------------------
8 # Copyright (C) 2008-2011 The IPython Development Team
8 # Copyright (C) 2008-2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-------------------------------------------------------------------------------
12 #-------------------------------------------------------------------------------
13
13
14 #-------------------------------------------------------------------------------
14 #-------------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-------------------------------------------------------------------------------
16 #-------------------------------------------------------------------------------
17
17
18 import copy
18 import copy
19 import logging
19 import logging
20 import sys
20 import sys
21 from types import FunctionType
21 from types import FunctionType
22
22
23 try:
23 try:
24 import cPickle as pickle
24 import cPickle as pickle
25 except ImportError:
25 except ImportError:
26 import pickle
26 import pickle
27
27
28 try:
28 try:
29 import numpy
29 import numpy
30 except:
30 except:
31 numpy = None
31 numpy = None
32
32
33 import codeutil
33 import codeutil
34 import py3compat
34 import py3compat
35 from importstring import import_item
35 from importstring import import_item
36
36
37 from IPython.config import Application
37 from IPython.config import Application
38
38
39 if py3compat.PY3:
39 if py3compat.PY3:
40 buffer = memoryview
40 buffer = memoryview
41 class_type = type
41 class_type = type
42 else:
42 else:
43 from types import ClassType
43 from types import ClassType
44 class_type = (type, ClassType)
44 class_type = (type, ClassType)
45
45
46 #-------------------------------------------------------------------------------
46 #-------------------------------------------------------------------------------
47 # Classes
47 # Classes
48 #-------------------------------------------------------------------------------
48 #-------------------------------------------------------------------------------
49
49
50
50
51 class CannedObject(object):
51 class CannedObject(object):
52 def __init__(self, obj, keys=[]):
52 def __init__(self, obj, keys=[]):
53 self.keys = keys
53 self.keys = keys
54 self.obj = copy.copy(obj)
54 self.obj = copy.copy(obj)
55 for key in keys:
55 for key in keys:
56 setattr(self.obj, key, can(getattr(obj, key)))
56 setattr(self.obj, key, can(getattr(obj, key)))
57
57
58 self.buffers = []
58 self.buffers = []
59
59
60 def get_object(self, g=None):
60 def get_object(self, g=None):
61 if g is None:
61 if g is None:
62 g = {}
62 g = {}
63 for key in self.keys:
63 for key in self.keys:
64 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
64 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
65 return self.obj
65 return self.obj
66
66
67
67
68 class Reference(CannedObject):
68 class Reference(CannedObject):
69 """object for wrapping a remote reference by name."""
69 """object for wrapping a remote reference by name."""
70 def __init__(self, name):
70 def __init__(self, name):
71 if not isinstance(name, basestring):
71 if not isinstance(name, basestring):
72 raise TypeError("illegal name: %r"%name)
72 raise TypeError("illegal name: %r"%name)
73 self.name = name
73 self.name = name
74 self.buffers = []
74 self.buffers = []
75
75
76 def __repr__(self):
76 def __repr__(self):
77 return "<Reference: %r>"%self.name
77 return "<Reference: %r>"%self.name
78
78
79 def get_object(self, g=None):
79 def get_object(self, g=None):
80 if g is None:
80 if g is None:
81 g = {}
81 g = {}
82
82
83 return eval(self.name, g)
83 return eval(self.name, g)
84
84
85
85
86 class CannedFunction(CannedObject):
86 class CannedFunction(CannedObject):
87
87
88 def __init__(self, f):
88 def __init__(self, f):
89 self._check_type(f)
89 self._check_type(f)
90 self.code = f.func_code
90 self.code = f.func_code
91 if f.func_defaults:
91 if f.func_defaults:
92 self.defaults = [ can(fd) for fd in f.func_defaults ]
92 self.defaults = [ can(fd) for fd in f.func_defaults ]
93 else:
93 else:
94 self.defaults = None
94 self.defaults = None
95 self.module = f.__module__ or '__main__'
95 self.module = f.__module__ or '__main__'
96 self.__name__ = f.__name__
96 self.__name__ = f.__name__
97 self.buffers = []
97 self.buffers = []
98
98
99 def _check_type(self, obj):
99 def _check_type(self, obj):
100 assert isinstance(obj, FunctionType), "Not a function type"
100 assert isinstance(obj, FunctionType), "Not a function type"
101
101
102 def get_object(self, g=None):
102 def get_object(self, g=None):
103 # try to load function back into its module:
103 # try to load function back into its module:
104 if not self.module.startswith('__'):
104 if not self.module.startswith('__'):
105 __import__(self.module)
105 __import__(self.module)
106 g = sys.modules[self.module].__dict__
106 g = sys.modules[self.module].__dict__
107
107
108 if g is None:
108 if g is None:
109 g = {}
109 g = {}
110 if self.defaults:
110 if self.defaults:
111 defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
111 defaults = tuple(uncan(cfd, g) for cfd in self.defaults)
112 else:
112 else:
113 defaults = None
113 defaults = None
114 newFunc = FunctionType(self.code, g, self.__name__, defaults)
114 newFunc = FunctionType(self.code, g, self.__name__, defaults)
115 return newFunc
115 return newFunc
116
116
117 class CannedClass(CannedObject):
117 class CannedClass(CannedObject):
118
118
119 def __init__(self, cls):
119 def __init__(self, cls):
120 self._check_type(cls)
120 self._check_type(cls)
121 self.name = cls.__name__
121 self.name = cls.__name__
122 self.old_style = not isinstance(cls, type)
122 self.old_style = not isinstance(cls, type)
123 self._canned_dict = {}
123 self._canned_dict = {}
124 for k,v in cls.__dict__.items():
124 for k,v in cls.__dict__.items():
125 if k not in ('__weakref__', '__dict__'):
125 if k not in ('__weakref__', '__dict__'):
126 self._canned_dict[k] = can(v)
126 self._canned_dict[k] = can(v)
127 if self.old_style:
127 if self.old_style:
128 mro = []
128 mro = []
129 else:
129 else:
130 mro = cls.mro()
130 mro = cls.mro()
131
131
132 self.parents = [ can(c) for c in mro[1:] ]
132 self.parents = [ can(c) for c in mro[1:] ]
133 self.buffers = []
133 self.buffers = []
134
134
135 def _check_type(self, obj):
135 def _check_type(self, obj):
136 assert isinstance(obj, class_type), "Not a class type"
136 assert isinstance(obj, class_type), "Not a class type"
137
137
138 def get_object(self, g=None):
138 def get_object(self, g=None):
139 parents = tuple(uncan(p, g) for p in self.parents)
139 parents = tuple(uncan(p, g) for p in self.parents)
140 return type(self.name, parents, uncan_dict(self._canned_dict, g=g))
140 return type(self.name, parents, uncan_dict(self._canned_dict, g=g))
141
141
142 class CannedArray(CannedObject):
142 class CannedArray(CannedObject):
143 def __init__(self, obj):
143 def __init__(self, obj):
144 self.shape = obj.shape
144 self.shape = obj.shape
145 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
145 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
146 if sum(obj.shape) == 0:
146 if sum(obj.shape) == 0:
147 # just pickle it
147 # just pickle it
148 self.buffers = [pickle.dumps(obj, -1)]
148 self.buffers = [pickle.dumps(obj, -1)]
149 else:
149 else:
150 # ensure contiguous
150 # ensure contiguous
151 obj = numpy.ascontiguousarray(obj, dtype=None)
151 obj = numpy.ascontiguousarray(obj, dtype=None)
152 self.buffers = [buffer(obj)]
152 self.buffers = [buffer(obj)]
153
153
154 def get_object(self, g=None):
154 def get_object(self, g=None):
155 data = self.buffers[0]
155 data = self.buffers[0]
156 if sum(self.shape) == 0:
156 if sum(self.shape) == 0:
157 # no shape, we just pickled it
157 # no shape, we just pickled it
158 return pickle.loads(data)
158 return pickle.loads(data)
159 else:
159 else:
160 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
160 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
161
161
162
162
163 class CannedBytes(CannedObject):
163 class CannedBytes(CannedObject):
164 wrap = bytes
164 wrap = bytes
165 def __init__(self, obj):
165 def __init__(self, obj):
166 self.buffers = [obj]
166 self.buffers = [obj]
167
167
168 def get_object(self, g=None):
168 def get_object(self, g=None):
169 data = self.buffers[0]
169 data = self.buffers[0]
170 return self.wrap(data)
170 return self.wrap(data)
171
171
172 def CannedBuffer(CannedBytes):
172 def CannedBuffer(CannedBytes):
173 wrap = buffer
173 wrap = buffer
174
174
175 #-------------------------------------------------------------------------------
175 #-------------------------------------------------------------------------------
176 # Functions
176 # Functions
177 #-------------------------------------------------------------------------------
177 #-------------------------------------------------------------------------------
178
178
179 def _logger():
179 def _logger():
180 """get the logger for the current Application
180 """get the logger for the current Application
181
181
182 the root logger will be used if no Application is running
182 the root logger will be used if no Application is running
183 """
183 """
184 if Application.initialized():
184 if Application.initialized():
185 logger = Application.instance().log
185 logger = Application.instance().log
186 else:
186 else:
187 logger = logging.getLogger()
187 logger = logging.getLogger()
188 if not logger.handlers:
188 if not logger.handlers:
189 logging.basicConfig()
189 logging.basicConfig()
190
190
191 return logger
191 return logger
192
192
193 def _import_mapping(mapping, original=None):
193 def _import_mapping(mapping, original=None):
194 """import any string-keys in a type mapping
194 """import any string-keys in a type mapping
195
195
196 """
196 """
197 log = _logger()
197 log = _logger()
198 log.debug("Importing canning map")
198 log.debug("Importing canning map")
199 for key,value in mapping.items():
199 for key,value in mapping.items():
200 if isinstance(key, basestring):
200 if isinstance(key, basestring):
201 try:
201 try:
202 cls = import_item(key)
202 cls = import_item(key)
203 except Exception:
203 except Exception:
204 if original and key not in original:
204 if original and key not in original:
205 # only message on user-added classes
205 # only message on user-added classes
206 log.error("cannning class not importable: %r", key, exc_info=True)
206 log.error("cannning class not importable: %r", key, exc_info=True)
207 mapping.pop(key)
207 mapping.pop(key)
208 else:
208 else:
209 mapping[cls] = mapping.pop(key)
209 mapping[cls] = mapping.pop(key)
210
210
211 def istype(obj, check):
212 """like isinstance(obj, check), but strict
213
214 This won't catch subclasses.
215 """
216 if isinstance(check, tuple):
217 for cls in check:
218 if type(obj) is cls:
219 return True
220 return False
221 else:
222 return type(obj) is check
223
211 def can(obj):
224 def can(obj):
212 """prepare an object for pickling"""
225 """prepare an object for pickling"""
213
226
214 import_needed = False
227 import_needed = False
215
228
216 for cls,canner in can_map.iteritems():
229 for cls,canner in can_map.iteritems():
217 if isinstance(cls, basestring):
230 if isinstance(cls, basestring):
218 import_needed = True
231 import_needed = True
219 break
232 break
220 elif type(obj) is cls:
233 elif istype(obj, cls):
221 return canner(obj)
234 return canner(obj)
222
235
223 if import_needed:
236 if import_needed:
224 # perform can_map imports, then try again
237 # perform can_map imports, then try again
225 # this will usually only happen once
238 # this will usually only happen once
226 _import_mapping(can_map, _original_can_map)
239 _import_mapping(can_map, _original_can_map)
227 return can(obj)
240 return can(obj)
228
241
229 return obj
242 return obj
230
243
231 def can_class(obj):
244 def can_class(obj):
232 if isinstance(obj, class_type) and obj.__module__ == '__main__':
245 if isinstance(obj, class_type) and obj.__module__ == '__main__':
233 return CannedClass(obj)
246 return CannedClass(obj)
234 else:
247 else:
235 return obj
248 return obj
236
249
237 def can_dict(obj):
250 def can_dict(obj):
238 """can the *values* of a dict"""
251 """can the *values* of a dict"""
239 if isinstance(obj, dict):
252 if isinstance(obj, dict):
240 newobj = {}
253 newobj = {}
241 for k, v in obj.iteritems():
254 for k, v in obj.iteritems():
242 newobj[k] = can(v)
255 newobj[k] = can(v)
243 return newobj
256 return newobj
244 else:
257 else:
245 return obj
258 return obj
246
259
247 def can_sequence(obj):
260 def can_sequence(obj):
248 """can the elements of a sequence"""
261 """can the elements of a sequence"""
249 if isinstance(obj, (list, tuple)):
262 if isinstance(obj, (list, tuple)):
250 t = type(obj)
263 t = type(obj)
251 return t([can(i) for i in obj])
264 return t([can(i) for i in obj])
252 else:
265 else:
253 return obj
266 return obj
254
267
255 def uncan(obj, g=None):
268 def uncan(obj, g=None):
256 """invert canning"""
269 """invert canning"""
257
270
258 import_needed = False
271 import_needed = False
259 for cls,uncanner in uncan_map.iteritems():
272 for cls,uncanner in uncan_map.iteritems():
260 if isinstance(cls, basestring):
273 if isinstance(cls, basestring):
261 import_needed = True
274 import_needed = True
262 break
275 break
263 elif isinstance(obj, cls):
276 elif isinstance(obj, cls):
264 return uncanner(obj, g)
277 return uncanner(obj, g)
265
278
266 if import_needed:
279 if import_needed:
267 # perform uncan_map imports, then try again
280 # perform uncan_map imports, then try again
268 # this will usually only happen once
281 # this will usually only happen once
269 _import_mapping(uncan_map, _original_uncan_map)
282 _import_mapping(uncan_map, _original_uncan_map)
270 return uncan(obj, g)
283 return uncan(obj, g)
271
284
272 return obj
285 return obj
273
286
274 def uncan_dict(obj, g=None):
287 def uncan_dict(obj, g=None):
275 if isinstance(obj, dict):
288 if isinstance(obj, dict):
276 newobj = {}
289 newobj = {}
277 for k, v in obj.iteritems():
290 for k, v in obj.iteritems():
278 newobj[k] = uncan(v,g)
291 newobj[k] = uncan(v,g)
279 return newobj
292 return newobj
280 else:
293 else:
281 return obj
294 return obj
282
295
283 def uncan_sequence(obj, g=None):
296 def uncan_sequence(obj, g=None):
284 if isinstance(obj, (list, tuple)):
297 if isinstance(obj, (list, tuple)):
285 t = type(obj)
298 t = type(obj)
286 return t([uncan(i,g) for i in obj])
299 return t([uncan(i,g) for i in obj])
287 else:
300 else:
288 return obj
301 return obj
289
302
290
303
291 #-------------------------------------------------------------------------------
304 #-------------------------------------------------------------------------------
292 # API dictionaries
305 # API dictionaries
293 #-------------------------------------------------------------------------------
306 #-------------------------------------------------------------------------------
294
307
295 # These dicts can be extended for custom serialization of new objects
308 # These dicts can be extended for custom serialization of new objects
296
309
297 can_map = {
310 can_map = {
298 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
311 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
299 'numpy.ndarray' : CannedArray,
312 'numpy.ndarray' : CannedArray,
300 FunctionType : CannedFunction,
313 FunctionType : CannedFunction,
301 bytes : CannedBytes,
314 bytes : CannedBytes,
302 buffer : CannedBuffer,
315 buffer : CannedBuffer,
303 class_type : can_class,
316 class_type : can_class,
304 }
317 }
305
318
306 uncan_map = {
319 uncan_map = {
307 CannedObject : lambda obj, g: obj.get_object(g),
320 CannedObject : lambda obj, g: obj.get_object(g),
308 }
321 }
309
322
310 # for use in _import_mapping:
323 # for use in _import_mapping:
311 _original_can_map = can_map.copy()
324 _original_can_map = can_map.copy()
312 _original_uncan_map = uncan_map.copy()
325 _original_uncan_map = uncan_map.copy()
General Comments 0
You need to be logged in to leave comments. Login now