##// END OF EJS Templates
use logger for canning import error
MinRK -
Show More
@@ -1,226 +1,237 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """Pickle related utilities. Perhaps this should be called 'can'."""
3 """Pickle related utilities. Perhaps this should be called 'can'."""
4
4
5 __docformat__ = "restructuredtext en"
5 __docformat__ = "restructuredtext en"
6
6
7 #-------------------------------------------------------------------------------
7 #-------------------------------------------------------------------------------
8 # Copyright (C) 2008-2011 The IPython Development Team
8 # Copyright (C) 2008-2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-------------------------------------------------------------------------------
12 #-------------------------------------------------------------------------------
13
13
14 #-------------------------------------------------------------------------------
14 #-------------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-------------------------------------------------------------------------------
16 #-------------------------------------------------------------------------------
17
17
18 import copy
18 import copy
19 import logging
19 import sys
20 import sys
20 from types import FunctionType
21 from types import FunctionType
21
22
22 try:
23 try:
23 import cPickle as pickle
24 import cPickle as pickle
24 except ImportError:
25 except ImportError:
25 import pickle
26 import pickle
26
27
27 try:
28 try:
28 import numpy
29 import numpy
29 except:
30 except:
30 numpy = None
31 numpy = None
31
32
32 import codeutil
33 import codeutil
33 import py3compat
34 import py3compat
34 from importstring import import_item
35 from importstring import import_item
35
36
37 from IPython.config import Application
38
36 if py3compat.PY3:
39 if py3compat.PY3:
37 buffer = memoryview
40 buffer = memoryview
38
41
39 #-------------------------------------------------------------------------------
42 #-------------------------------------------------------------------------------
40 # Classes
43 # Classes
41 #-------------------------------------------------------------------------------
44 #-------------------------------------------------------------------------------
42
45
43
46
44 class CannedObject(object):
47 class CannedObject(object):
45 def __init__(self, obj, keys=[]):
48 def __init__(self, obj, keys=[]):
46 self.keys = keys
49 self.keys = keys
47 self.obj = copy.copy(obj)
50 self.obj = copy.copy(obj)
48 for key in keys:
51 for key in keys:
49 setattr(self.obj, key, can(getattr(obj, key)))
52 setattr(self.obj, key, can(getattr(obj, key)))
50
53
51 self.buffers = []
54 self.buffers = []
52
55
53 def get_object(self, g=None):
56 def get_object(self, g=None):
54 if g is None:
57 if g is None:
55 g = {}
58 g = {}
56 for key in self.keys:
59 for key in self.keys:
57 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
60 setattr(self.obj, key, uncan(getattr(self.obj, key), g))
58 return self.obj
61 return self.obj
59
62
60
63
61 class Reference(CannedObject):
64 class Reference(CannedObject):
62 """object for wrapping a remote reference by name."""
65 """object for wrapping a remote reference by name."""
63 def __init__(self, name):
66 def __init__(self, name):
64 if not isinstance(name, basestring):
67 if not isinstance(name, basestring):
65 raise TypeError("illegal name: %r"%name)
68 raise TypeError("illegal name: %r"%name)
66 self.name = name
69 self.name = name
67 self.buffers = []
70 self.buffers = []
68
71
69 def __repr__(self):
72 def __repr__(self):
70 return "<Reference: %r>"%self.name
73 return "<Reference: %r>"%self.name
71
74
72 def get_object(self, g=None):
75 def get_object(self, g=None):
73 if g is None:
76 if g is None:
74 g = {}
77 g = {}
75
78
76 return eval(self.name, g)
79 return eval(self.name, g)
77
80
78
81
79 class CannedFunction(CannedObject):
82 class CannedFunction(CannedObject):
80
83
81 def __init__(self, f):
84 def __init__(self, f):
82 self._check_type(f)
85 self._check_type(f)
83 self.code = f.func_code
86 self.code = f.func_code
84 self.defaults = f.func_defaults
87 self.defaults = f.func_defaults
85 self.module = f.__module__ or '__main__'
88 self.module = f.__module__ or '__main__'
86 self.__name__ = f.__name__
89 self.__name__ = f.__name__
87 self.buffers = []
90 self.buffers = []
88
91
89 def _check_type(self, obj):
92 def _check_type(self, obj):
90 assert isinstance(obj, FunctionType), "Not a function type"
93 assert isinstance(obj, FunctionType), "Not a function type"
91
94
92 def get_object(self, g=None):
95 def get_object(self, g=None):
93 # try to load function back into its module:
96 # try to load function back into its module:
94 if not self.module.startswith('__'):
97 if not self.module.startswith('__'):
95 try:
98 try:
96 __import__(self.module)
99 __import__(self.module)
97 except ImportError:
100 except ImportError:
98 pass
101 pass
99 else:
102 else:
100 g = sys.modules[self.module].__dict__
103 g = sys.modules[self.module].__dict__
101
104
102 if g is None:
105 if g is None:
103 g = {}
106 g = {}
104 newFunc = FunctionType(self.code, g, self.__name__, self.defaults)
107 newFunc = FunctionType(self.code, g, self.__name__, self.defaults)
105 return newFunc
108 return newFunc
106
109
107
110
108 class CannedArray(CannedObject):
111 class CannedArray(CannedObject):
109 def __init__(self, obj):
112 def __init__(self, obj):
110 self.shape = obj.shape
113 self.shape = obj.shape
111 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
114 self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str
112 if sum(obj.shape) == 0:
115 if sum(obj.shape) == 0:
113 # just pickle it
116 # just pickle it
114 self.buffers = [pickle.dumps(obj, -1)]
117 self.buffers = [pickle.dumps(obj, -1)]
115 else:
118 else:
116 # ensure contiguous
119 # ensure contiguous
117 obj = numpy.ascontiguousarray(obj, dtype=None)
120 obj = numpy.ascontiguousarray(obj, dtype=None)
118 self.buffers = [buffer(obj)]
121 self.buffers = [buffer(obj)]
119
122
120 def get_object(self, g=None):
123 def get_object(self, g=None):
121 data = self.buffers[0]
124 data = self.buffers[0]
122 if sum(self.shape) == 0:
125 if sum(self.shape) == 0:
123 # no shape, we just pickled it
126 # no shape, we just pickled it
124 return pickle.loads(data)
127 return pickle.loads(data)
125 else:
128 else:
126 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
129 return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape)
127
130
128
131
129 class CannedBytes(CannedObject):
132 class CannedBytes(CannedObject):
130 wrap = bytes
133 wrap = bytes
131 def __init__(self, obj):
134 def __init__(self, obj):
132 self.buffers = [obj]
135 self.buffers = [obj]
133
136
134 def get_object(self, g=None):
137 def get_object(self, g=None):
135 data = self.buffers[0]
138 data = self.buffers[0]
136 return self.wrap(data)
139 return self.wrap(data)
137
140
138 def CannedBuffer(CannedBytes):
141 def CannedBuffer(CannedBytes):
139 wrap = buffer
142 wrap = buffer
140
143
141 #-------------------------------------------------------------------------------
144 #-------------------------------------------------------------------------------
142 # Functions
145 # Functions
143 #-------------------------------------------------------------------------------
146 #-------------------------------------------------------------------------------
144
147
148 def _error(*args, **kwargs):
149 if Application.initialized():
150 logger = Application.instance().log
151 else:
152 logger = logging.getLogger()
153 if not logger.handlers:
154 logging.basicConfig()
155 logger.error(*args, **kwargs)
145
156
146 def can(obj):
157 def can(obj):
147 """prepare an object for pickling"""
158 """prepare an object for pickling"""
148 for cls,canner in can_map.iteritems():
159 for cls,canner in can_map.iteritems():
149 if isinstance(cls, basestring):
160 if isinstance(cls, basestring):
150 try:
161 try:
151 cls = import_item(cls)
162 cls = import_item(cls)
152 except Exception:
163 except Exception:
153 # not importable
164 _error("cannning class not importable: %r", cls, exc_info=True)
154 print "not importable: %r" % cls
165 cls = None
155 continue
166 continue
156 if isinstance(obj, cls):
167 if isinstance(obj, cls):
157 return canner(obj)
168 return canner(obj)
158 return obj
169 return obj
159
170
160 def can_dict(obj):
171 def can_dict(obj):
161 """can the *values* of a dict"""
172 """can the *values* of a dict"""
162 if isinstance(obj, dict):
173 if isinstance(obj, dict):
163 newobj = {}
174 newobj = {}
164 for k, v in obj.iteritems():
175 for k, v in obj.iteritems():
165 newobj[k] = can(v)
176 newobj[k] = can(v)
166 return newobj
177 return newobj
167 else:
178 else:
168 return obj
179 return obj
169
180
170 def can_sequence(obj):
181 def can_sequence(obj):
171 """can the elements of a sequence"""
182 """can the elements of a sequence"""
172 if isinstance(obj, (list, tuple)):
183 if isinstance(obj, (list, tuple)):
173 t = type(obj)
184 t = type(obj)
174 return t([can(i) for i in obj])
185 return t([can(i) for i in obj])
175 else:
186 else:
176 return obj
187 return obj
177
188
178 def uncan(obj, g=None):
189 def uncan(obj, g=None):
179 """invert canning"""
190 """invert canning"""
180 for cls,uncanner in uncan_map.iteritems():
191 for cls,uncanner in uncan_map.iteritems():
181 if isinstance(cls, basestring):
192 if isinstance(cls, basestring):
182 try:
193 try:
183 cls = import_item(cls)
194 cls = import_item(cls)
184 except Exception:
195 except Exception:
185 # not importable
196 _error("uncanning class not importable: %r", cls, exc_info=True)
186 print "not importable: %r" % cls
197 cls = None
187 continue
198 continue
188 if isinstance(obj, cls):
199 if isinstance(obj, cls):
189 return uncanner(obj, g)
200 return uncanner(obj, g)
190 return obj
201 return obj
191
202
192 def uncan_dict(obj, g=None):
203 def uncan_dict(obj, g=None):
193 if isinstance(obj, dict):
204 if isinstance(obj, dict):
194 newobj = {}
205 newobj = {}
195 for k, v in obj.iteritems():
206 for k, v in obj.iteritems():
196 newobj[k] = uncan(v,g)
207 newobj[k] = uncan(v,g)
197 return newobj
208 return newobj
198 else:
209 else:
199 return obj
210 return obj
200
211
201 def uncan_sequence(obj, g=None):
212 def uncan_sequence(obj, g=None):
202 if isinstance(obj, (list, tuple)):
213 if isinstance(obj, (list, tuple)):
203 t = type(obj)
214 t = type(obj)
204 return t([uncan(i,g) for i in obj])
215 return t([uncan(i,g) for i in obj])
205 else:
216 else:
206 return obj
217 return obj
207
218
208
219
209 #-------------------------------------------------------------------------------
220 #-------------------------------------------------------------------------------
210 # API dictionary
221 # API dictionary
211 #-------------------------------------------------------------------------------
222 #-------------------------------------------------------------------------------
212
223
213 # These dicts can be extended for custom serialization of new objects
224 # These dicts can be extended for custom serialization of new objects
214
225
215 can_map = {
226 can_map = {
216 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
227 'IPython.parallel.dependent' : lambda obj: CannedObject(obj, keys=('f','df')),
217 'numpy.ndarray' : CannedArray,
228 'numpy.ndarray' : CannedArray,
218 FunctionType : CannedFunction,
229 FunctionType : CannedFunction,
219 bytes : CannedBytes,
230 bytes : CannedBytes,
220 buffer : CannedBuffer,
231 buffer : CannedBuffer,
221 }
232 }
222
233
223 uncan_map = {
234 uncan_map = {
224 CannedObject : lambda obj, g: obj.get_object(g),
235 CannedObject : lambda obj, g: obj.get_object(g),
225 }
236 }
226
237
General Comments 0
You need to be logged in to leave comments. Login now