Show More
@@ -0,0 +1,39 b'' | |||||
|
1 | # encoding: utf-8 | |||
|
2 | ||||
|
3 | """Utilities to enable code objects to be pickled. | |||
|
4 | ||||
|
5 | Any process that import this module will be able to pickle code objects. This | |||
|
6 | includes the func_code attribute of any function. Once unpickled, new | |||
|
7 | functions can be built using new.function(code, globals()). Eventually | |||
|
8 | we need to automate all of this so that functions themselves can be pickled. | |||
|
9 | ||||
|
10 | Reference: A. Tremols, P Cogolo, "Python Cookbook," p 302-305 | |||
|
11 | """ | |||
|
12 | ||||
|
13 | __docformat__ = "restructuredtext en" | |||
|
14 | ||||
|
15 | #------------------------------------------------------------------------------- | |||
|
16 | # Copyright (C) 2008 The IPython Development Team | |||
|
17 | # | |||
|
18 | # Distributed under the terms of the BSD License. The full license is in | |||
|
19 | # the file COPYING, distributed as part of this software. | |||
|
20 | #------------------------------------------------------------------------------- | |||
|
21 | ||||
|
22 | #------------------------------------------------------------------------------- | |||
|
23 | # Imports | |||
|
24 | #------------------------------------------------------------------------------- | |||
|
25 | ||||
|
26 | import new, types, copy_reg | |||
|
27 | ||||
|
28 | def code_ctor(*args): | |||
|
29 | return new.code(*args) | |||
|
30 | ||||
|
31 | def reduce_code(co): | |||
|
32 | if co.co_freevars or co.co_cellvars: | |||
|
33 | raise ValueError("Sorry, cannot pickle code objects with closures") | |||
|
34 | return code_ctor, (co.co_argcount, co.co_nlocals, co.co_stacksize, | |||
|
35 | co.co_flags, co.co_code, co.co_consts, co.co_names, | |||
|
36 | co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, | |||
|
37 | co.co_lnotab) | |||
|
38 | ||||
|
39 | copy_reg.pickle(types.CodeType, reduce_code) No newline at end of file |
@@ -1,167 +1,165 b'' | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- |
|
2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- | |
3 |
|
3 | |||
4 | """Refactored serialization classes and interfaces.""" |
|
4 | """Refactored serialization classes and interfaces.""" | |
5 |
|
5 | |||
6 | __docformat__ = "restructuredtext en" |
|
6 | __docformat__ = "restructuredtext en" | |
7 |
|
7 | |||
8 | # Tell nose to skip this module |
|
8 | # Tell nose to skip this module | |
9 | __test__ = {} |
|
9 | __test__ = {} | |
10 |
|
10 | |||
11 | #------------------------------------------------------------------------------- |
|
11 | #------------------------------------------------------------------------------- | |
12 | # Copyright (C) 2008 The IPython Development Team |
|
12 | # Copyright (C) 2008 The IPython Development Team | |
13 | # |
|
13 | # | |
14 | # Distributed under the terms of the BSD License. The full license is in |
|
14 | # Distributed under the terms of the BSD License. The full license is in | |
15 | # the file COPYING, distributed as part of this software. |
|
15 | # the file COPYING, distributed as part of this software. | |
16 | #------------------------------------------------------------------------------- |
|
16 | #------------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | #------------------------------------------------------------------------------- |
|
18 | #------------------------------------------------------------------------------- | |
19 | # Imports |
|
19 | # Imports | |
20 | #------------------------------------------------------------------------------- |
|
20 | #------------------------------------------------------------------------------- | |
21 |
|
21 | |||
22 | import cPickle as pickle |
|
22 | import cPickle as pickle | |
23 |
|
23 | |||
24 | # from twisted.python import components |
|
|||
25 | # from zope.interface import Interface, implements |
|
|||
26 |
|
||||
27 | try: |
|
24 | try: | |
28 | import numpy |
|
25 | import numpy | |
29 | except ImportError: |
|
26 | except ImportError: | |
30 | pass |
|
27 | pass | |
31 |
|
28 | |||
32 | from IPython.kernel.error import SerializationError |
|
29 | class SerializationError(Exception): | |
|
30 | pass | |||
33 |
|
31 | |||
34 | #----------------------------------------------------------------------------- |
|
32 | #----------------------------------------------------------------------------- | |
35 | # Classes and functions |
|
33 | # Classes and functions | |
36 | #----------------------------------------------------------------------------- |
|
34 | #----------------------------------------------------------------------------- | |
37 |
|
35 | |||
38 | class ISerialized: |
|
36 | class ISerialized: | |
39 |
|
37 | |||
40 | def getData(): |
|
38 | def getData(): | |
41 | """""" |
|
39 | """""" | |
42 |
|
40 | |||
43 | def getDataSize(units=10.0**6): |
|
41 | def getDataSize(units=10.0**6): | |
44 | """""" |
|
42 | """""" | |
45 |
|
43 | |||
46 | def getTypeDescriptor(): |
|
44 | def getTypeDescriptor(): | |
47 | """""" |
|
45 | """""" | |
48 |
|
46 | |||
49 | def getMetadata(): |
|
47 | def getMetadata(): | |
50 | """""" |
|
48 | """""" | |
51 |
|
49 | |||
52 |
|
50 | |||
53 | class IUnSerialized: |
|
51 | class IUnSerialized: | |
54 |
|
52 | |||
55 | def getObject(): |
|
53 | def getObject(): | |
56 | """""" |
|
54 | """""" | |
57 |
|
55 | |||
58 | class Serialized(object): |
|
56 | class Serialized(object): | |
59 |
|
57 | |||
60 | # implements(ISerialized) |
|
58 | # implements(ISerialized) | |
61 |
|
59 | |||
62 | def __init__(self, data, typeDescriptor, metadata={}): |
|
60 | def __init__(self, data, typeDescriptor, metadata={}): | |
63 | self.data = data |
|
61 | self.data = data | |
64 | self.typeDescriptor = typeDescriptor |
|
62 | self.typeDescriptor = typeDescriptor | |
65 | self.metadata = metadata |
|
63 | self.metadata = metadata | |
66 |
|
64 | |||
67 | def getData(self): |
|
65 | def getData(self): | |
68 | return self.data |
|
66 | return self.data | |
69 |
|
67 | |||
70 | def getDataSize(self, units=10.0**6): |
|
68 | def getDataSize(self, units=10.0**6): | |
71 | return len(self.data)/units |
|
69 | return len(self.data)/units | |
72 |
|
70 | |||
73 | def getTypeDescriptor(self): |
|
71 | def getTypeDescriptor(self): | |
74 | return self.typeDescriptor |
|
72 | return self.typeDescriptor | |
75 |
|
73 | |||
76 | def getMetadata(self): |
|
74 | def getMetadata(self): | |
77 | return self.metadata |
|
75 | return self.metadata | |
78 |
|
76 | |||
79 |
|
77 | |||
80 | class UnSerialized(object): |
|
78 | class UnSerialized(object): | |
81 |
|
79 | |||
82 | # implements(IUnSerialized) |
|
80 | # implements(IUnSerialized) | |
83 |
|
81 | |||
84 | def __init__(self, obj): |
|
82 | def __init__(self, obj): | |
85 | self.obj = obj |
|
83 | self.obj = obj | |
86 |
|
84 | |||
87 | def getObject(self): |
|
85 | def getObject(self): | |
88 | return self.obj |
|
86 | return self.obj | |
89 |
|
87 | |||
90 |
|
88 | |||
91 | class SerializeIt(object): |
|
89 | class SerializeIt(object): | |
92 |
|
90 | |||
93 | # implements(ISerialized) |
|
91 | # implements(ISerialized) | |
94 |
|
92 | |||
95 | def __init__(self, unSerialized): |
|
93 | def __init__(self, unSerialized): | |
96 | self.data = None |
|
94 | self.data = None | |
97 | self.obj = unSerialized.getObject() |
|
95 | self.obj = unSerialized.getObject() | |
98 | if globals().has_key('numpy') and isinstance(self.obj, numpy.ndarray): |
|
96 | if globals().has_key('numpy') and isinstance(self.obj, numpy.ndarray): | |
99 | if len(self.obj) == 0: # length 0 arrays can't be reconstructed |
|
97 | if len(self.obj) == 0: # length 0 arrays can't be reconstructed | |
100 | raise SerializationError("You cannot send a length 0 array") |
|
98 | raise SerializationError("You cannot send a length 0 array") | |
101 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) |
|
99 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) | |
102 | self.typeDescriptor = 'ndarray' |
|
100 | self.typeDescriptor = 'ndarray' | |
103 | self.metadata = {'shape':self.obj.shape, |
|
101 | self.metadata = {'shape':self.obj.shape, | |
104 | 'dtype':self.obj.dtype.str} |
|
102 | 'dtype':self.obj.dtype.str} | |
105 | elif isinstance(self.obj, str): |
|
103 | elif isinstance(self.obj, str): | |
106 | self.typeDescriptor = 'bytes' |
|
104 | self.typeDescriptor = 'bytes' | |
107 | self.metadata = {} |
|
105 | self.metadata = {} | |
108 | elif isinstance(self.obj, buffer): |
|
106 | elif isinstance(self.obj, buffer): | |
109 | self.typeDescriptor = 'buffer' |
|
107 | self.typeDescriptor = 'buffer' | |
110 | self.metadata = {} |
|
108 | self.metadata = {} | |
111 | else: |
|
109 | else: | |
112 | self.typeDescriptor = 'pickle' |
|
110 | self.typeDescriptor = 'pickle' | |
113 | self.metadata = {} |
|
111 | self.metadata = {} | |
114 | self._generateData() |
|
112 | self._generateData() | |
115 |
|
113 | |||
116 | def _generateData(self): |
|
114 | def _generateData(self): | |
117 | if self.typeDescriptor == 'ndarray': |
|
115 | if self.typeDescriptor == 'ndarray': | |
118 | self.data = numpy.getbuffer(self.obj) |
|
116 | self.data = numpy.getbuffer(self.obj) | |
119 | elif self.typeDescriptor in ('bytes', 'buffer'): |
|
117 | elif self.typeDescriptor in ('bytes', 'buffer'): | |
120 | self.data = self.obj |
|
118 | self.data = self.obj | |
121 | elif self.typeDescriptor == 'pickle': |
|
119 | elif self.typeDescriptor == 'pickle': | |
122 | self.data = pickle.dumps(self.obj, -1) |
|
120 | self.data = pickle.dumps(self.obj, -1) | |
123 | else: |
|
121 | else: | |
124 | raise SerializationError("Really wierd serialization error.") |
|
122 | raise SerializationError("Really wierd serialization error.") | |
125 | del self.obj |
|
123 | del self.obj | |
126 |
|
124 | |||
127 | def getData(self): |
|
125 | def getData(self): | |
128 | return self.data |
|
126 | return self.data | |
129 |
|
127 | |||
130 | def getDataSize(self, units=10.0**6): |
|
128 | def getDataSize(self, units=10.0**6): | |
131 | return 1.0*len(self.data)/units |
|
129 | return 1.0*len(self.data)/units | |
132 |
|
130 | |||
133 | def getTypeDescriptor(self): |
|
131 | def getTypeDescriptor(self): | |
134 | return self.typeDescriptor |
|
132 | return self.typeDescriptor | |
135 |
|
133 | |||
136 | def getMetadata(self): |
|
134 | def getMetadata(self): | |
137 | return self.metadata |
|
135 | return self.metadata | |
138 |
|
136 | |||
139 |
|
137 | |||
140 | class UnSerializeIt(UnSerialized): |
|
138 | class UnSerializeIt(UnSerialized): | |
141 |
|
139 | |||
142 | # implements(IUnSerialized) |
|
140 | # implements(IUnSerialized) | |
143 |
|
141 | |||
144 | def __init__(self, serialized): |
|
142 | def __init__(self, serialized): | |
145 | self.serialized = serialized |
|
143 | self.serialized = serialized | |
146 |
|
144 | |||
147 | def getObject(self): |
|
145 | def getObject(self): | |
148 | typeDescriptor = self.serialized.getTypeDescriptor() |
|
146 | typeDescriptor = self.serialized.getTypeDescriptor() | |
149 | if globals().has_key('numpy') and typeDescriptor == 'ndarray': |
|
147 | if globals().has_key('numpy') and typeDescriptor == 'ndarray': | |
150 | result = numpy.frombuffer(self.serialized.getData(), dtype = self.serialized.metadata['dtype']) |
|
148 | result = numpy.frombuffer(self.serialized.getData(), dtype = self.serialized.metadata['dtype']) | |
151 | result.shape = self.serialized.metadata['shape'] |
|
149 | result.shape = self.serialized.metadata['shape'] | |
152 | # This is a hack to make the array writable. We are working with |
|
150 | # This is a hack to make the array writable. We are working with | |
153 | # the numpy folks to address this issue. |
|
151 | # the numpy folks to address this issue. | |
154 | result = result.copy() |
|
152 | result = result.copy() | |
155 | elif typeDescriptor == 'pickle': |
|
153 | elif typeDescriptor == 'pickle': | |
156 | result = pickle.loads(self.serialized.getData()) |
|
154 | result = pickle.loads(self.serialized.getData()) | |
157 | elif typeDescriptor in ('bytes', 'buffer'): |
|
155 | elif typeDescriptor in ('bytes', 'buffer'): | |
158 | result = self.serialized.getData() |
|
156 | result = self.serialized.getData() | |
159 | else: |
|
157 | else: | |
160 | raise SerializationError("Really wierd serialization error.") |
|
158 | raise SerializationError("Really wierd serialization error.") | |
161 | return result |
|
159 | return result | |
162 |
|
160 | |||
163 | def serialize(obj): |
|
161 | def serialize(obj): | |
164 | return SerializeIt(UnSerialized(obj)) |
|
162 | return SerializeIt(UnSerialized(obj)) | |
165 |
|
163 | |||
166 | def unserialize(serialized): |
|
164 | def unserialize(serialized): | |
167 | return UnSerializeIt(serialized).getObject() |
|
165 | return UnSerializeIt(serialized).getObject() |
@@ -1,115 +1,115 b'' | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 |
|
2 | |||
3 | """Pickle related utilities. Perhaps this should be called 'can'.""" |
|
3 | """Pickle related utilities. Perhaps this should be called 'can'.""" | |
4 |
|
4 | |||
5 | __docformat__ = "restructuredtext en" |
|
5 | __docformat__ = "restructuredtext en" | |
6 |
|
6 | |||
7 | #------------------------------------------------------------------------------- |
|
7 | #------------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008 The IPython Development Team |
|
8 | # Copyright (C) 2008 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #------------------------------------------------------------------------------- |
|
12 | #------------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #------------------------------------------------------------------------------- |
|
14 | #------------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #------------------------------------------------------------------------------- |
|
16 | #------------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | from types import FunctionType |
|
18 | from types import FunctionType | |
19 |
|
19 | |||
20 | # contents of codeutil should either be in here, or codeutil belongs in IPython/util |
|
20 | # contents of codeutil should either be in here, or codeutil belongs in IPython/util | |
21 | from IPython.kernel import codeutil |
|
|||
22 | from IPython.zmq.parallel.dependency import dependent |
|
21 | from IPython.zmq.parallel.dependency import dependent | |
|
22 | import codeutil | |||
23 |
|
23 | |||
24 | class CannedObject(object): |
|
24 | class CannedObject(object): | |
25 | def __init__(self, obj, keys=[]): |
|
25 | def __init__(self, obj, keys=[]): | |
26 | self.keys = keys |
|
26 | self.keys = keys | |
27 | self.obj = obj |
|
27 | self.obj = obj | |
28 | for key in keys: |
|
28 | for key in keys: | |
29 | setattr(obj, key, can(getattr(obj, key))) |
|
29 | setattr(obj, key, can(getattr(obj, key))) | |
30 |
|
30 | |||
31 |
|
31 | |||
32 | def getObject(self, g=None): |
|
32 | def getObject(self, g=None): | |
33 | if g is None: |
|
33 | if g is None: | |
34 | g = globals() |
|
34 | g = globals() | |
35 | for key in self.keys: |
|
35 | for key in self.keys: | |
36 | setattr(self.obj, key, uncan(getattr(self.obj, key), g)) |
|
36 | setattr(self.obj, key, uncan(getattr(self.obj, key), g)) | |
37 | return self.obj |
|
37 | return self.obj | |
38 |
|
38 | |||
39 |
|
39 | |||
40 |
|
40 | |||
41 | class CannedFunction(CannedObject): |
|
41 | class CannedFunction(CannedObject): | |
42 |
|
42 | |||
43 | def __init__(self, f): |
|
43 | def __init__(self, f): | |
44 | self._checkType(f) |
|
44 | self._checkType(f) | |
45 | self.code = f.func_code |
|
45 | self.code = f.func_code | |
46 |
|
46 | |||
47 | def _checkType(self, obj): |
|
47 | def _checkType(self, obj): | |
48 | assert isinstance(obj, FunctionType), "Not a function type" |
|
48 | assert isinstance(obj, FunctionType), "Not a function type" | |
49 |
|
49 | |||
50 | def getFunction(self, g=None): |
|
50 | def getFunction(self, g=None): | |
51 | if g is None: |
|
51 | if g is None: | |
52 | g = globals() |
|
52 | g = globals() | |
53 | newFunc = FunctionType(self.code, g) |
|
53 | newFunc = FunctionType(self.code, g) | |
54 | return newFunc |
|
54 | return newFunc | |
55 |
|
55 | |||
56 | def can(obj): |
|
56 | def can(obj): | |
57 | if isinstance(obj, FunctionType): |
|
57 | if isinstance(obj, FunctionType): | |
58 | return CannedFunction(obj) |
|
58 | return CannedFunction(obj) | |
59 | elif isinstance(obj, dependent): |
|
59 | elif isinstance(obj, dependent): | |
60 | keys = ('f','df') |
|
60 | keys = ('f','df') | |
61 | return CannedObject(obj, keys=keys) |
|
61 | return CannedObject(obj, keys=keys) | |
62 | elif isinstance(obj,dict): |
|
62 | elif isinstance(obj,dict): | |
63 | return canDict(obj) |
|
63 | return canDict(obj) | |
64 | elif isinstance(obj, (list,tuple)): |
|
64 | elif isinstance(obj, (list,tuple)): | |
65 | return canSequence(obj) |
|
65 | return canSequence(obj) | |
66 | else: |
|
66 | else: | |
67 | return obj |
|
67 | return obj | |
68 |
|
68 | |||
69 | def canDict(obj): |
|
69 | def canDict(obj): | |
70 | if isinstance(obj, dict): |
|
70 | if isinstance(obj, dict): | |
71 | newobj = {} |
|
71 | newobj = {} | |
72 | for k, v in obj.iteritems(): |
|
72 | for k, v in obj.iteritems(): | |
73 | newobj[k] = can(v) |
|
73 | newobj[k] = can(v) | |
74 | return newobj |
|
74 | return newobj | |
75 | else: |
|
75 | else: | |
76 | return obj |
|
76 | return obj | |
77 |
|
77 | |||
78 | def canSequence(obj): |
|
78 | def canSequence(obj): | |
79 | if isinstance(obj, (list, tuple)): |
|
79 | if isinstance(obj, (list, tuple)): | |
80 | t = type(obj) |
|
80 | t = type(obj) | |
81 | return t([can(i) for i in obj]) |
|
81 | return t([can(i) for i in obj]) | |
82 | else: |
|
82 | else: | |
83 | return obj |
|
83 | return obj | |
84 |
|
84 | |||
85 | def uncan(obj, g=None): |
|
85 | def uncan(obj, g=None): | |
86 | if isinstance(obj, CannedFunction): |
|
86 | if isinstance(obj, CannedFunction): | |
87 | return obj.getFunction(g) |
|
87 | return obj.getFunction(g) | |
88 | elif isinstance(obj, CannedObject): |
|
88 | elif isinstance(obj, CannedObject): | |
89 | return obj.getObject(g) |
|
89 | return obj.getObject(g) | |
90 | elif isinstance(obj,dict): |
|
90 | elif isinstance(obj,dict): | |
91 | return uncanDict(obj) |
|
91 | return uncanDict(obj) | |
92 | elif isinstance(obj, (list,tuple)): |
|
92 | elif isinstance(obj, (list,tuple)): | |
93 | return uncanSequence(obj) |
|
93 | return uncanSequence(obj) | |
94 | else: |
|
94 | else: | |
95 | return obj |
|
95 | return obj | |
96 |
|
96 | |||
97 | def uncanDict(obj, g=None): |
|
97 | def uncanDict(obj, g=None): | |
98 | if isinstance(obj, dict): |
|
98 | if isinstance(obj, dict): | |
99 | newobj = {} |
|
99 | newobj = {} | |
100 | for k, v in obj.iteritems(): |
|
100 | for k, v in obj.iteritems(): | |
101 | newobj[k] = uncan(v,g) |
|
101 | newobj[k] = uncan(v,g) | |
102 | return newobj |
|
102 | return newobj | |
103 | else: |
|
103 | else: | |
104 | return obj |
|
104 | return obj | |
105 |
|
105 | |||
106 | def uncanSequence(obj, g=None): |
|
106 | def uncanSequence(obj, g=None): | |
107 | if isinstance(obj, (list, tuple)): |
|
107 | if isinstance(obj, (list, tuple)): | |
108 | t = type(obj) |
|
108 | t = type(obj) | |
109 | return t([uncan(i,g) for i in obj]) |
|
109 | return t([uncan(i,g) for i in obj]) | |
110 | else: |
|
110 | else: | |
111 | return obj |
|
111 | return obj | |
112 |
|
112 | |||
113 |
|
113 | |||
114 | def rebindFunctionGlobals(f, glbls): |
|
114 | def rebindFunctionGlobals(f, glbls): | |
115 | return FunctionType(f.func_code, glbls) |
|
115 | return FunctionType(f.func_code, glbls) |
General Comments 0
You need to be logged in to leave comments.
Login now