Show More
@@ -1,177 +1,177 b'' | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- |
|
2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- | |
3 |
|
3 | |||
4 | """Refactored serialization classes and interfaces.""" |
|
4 | """Refactored serialization classes and interfaces.""" | |
5 |
|
5 | |||
6 | __docformat__ = "restructuredtext en" |
|
6 | __docformat__ = "restructuredtext en" | |
7 |
|
7 | |||
8 | # Tell nose to skip this module |
|
8 | # Tell nose to skip this module | |
9 | __test__ = {} |
|
9 | __test__ = {} | |
10 |
|
10 | |||
11 | #------------------------------------------------------------------------------- |
|
11 | #------------------------------------------------------------------------------- | |
12 | # Copyright (C) 2008-2011 The IPython Development Team |
|
12 | # Copyright (C) 2008-2011 The IPython Development Team | |
13 | # |
|
13 | # | |
14 | # Distributed under the terms of the BSD License. The full license is in |
|
14 | # Distributed under the terms of the BSD License. The full license is in | |
15 | # the file COPYING, distributed as part of this software. |
|
15 | # the file COPYING, distributed as part of this software. | |
16 | #------------------------------------------------------------------------------- |
|
16 | #------------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | #------------------------------------------------------------------------------- |
|
18 | #------------------------------------------------------------------------------- | |
19 | # Imports |
|
19 | # Imports | |
20 | #------------------------------------------------------------------------------- |
|
20 | #------------------------------------------------------------------------------- | |
21 |
|
21 | |||
22 | import sys |
|
22 | import sys | |
23 | import cPickle as pickle |
|
23 | import cPickle as pickle | |
24 |
|
24 | |||
25 | try: |
|
25 | try: | |
26 | import numpy |
|
26 | import numpy | |
27 | except ImportError: |
|
27 | except ImportError: | |
28 | numpy = None |
|
28 | numpy = None | |
29 |
|
29 | |||
30 | class SerializationError(Exception): |
|
30 | class SerializationError(Exception): | |
31 | pass |
|
31 | pass | |
32 |
|
32 | |||
33 | if sys.version_info[0] >= 3: |
|
33 | if sys.version_info[0] >= 3: | |
34 | buffer = memoryview |
|
34 | buffer = memoryview | |
35 | py3k = True |
|
35 | py3k = True | |
36 | else: |
|
36 | else: | |
37 | py3k = False |
|
37 | py3k = False | |
38 | if sys.version_info[:2] <= (2,6): |
|
38 | if sys.version_info[:2] <= (2,6): | |
39 | memoryview = buffer |
|
39 | memoryview = buffer | |
40 |
|
40 | |||
41 | #----------------------------------------------------------------------------- |
|
41 | #----------------------------------------------------------------------------- | |
42 | # Classes and functions |
|
42 | # Classes and functions | |
43 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
44 |
|
44 | |||
45 | class ISerialized: |
|
45 | class ISerialized: | |
46 |
|
46 | |||
47 | def getData(): |
|
47 | def getData(): | |
48 | """""" |
|
48 | """""" | |
49 |
|
49 | |||
50 | def getDataSize(units=10.0**6): |
|
50 | def getDataSize(units=10.0**6): | |
51 | """""" |
|
51 | """""" | |
52 |
|
52 | |||
53 | def getTypeDescriptor(): |
|
53 | def getTypeDescriptor(): | |
54 | """""" |
|
54 | """""" | |
55 |
|
55 | |||
56 | def getMetadata(): |
|
56 | def getMetadata(): | |
57 | """""" |
|
57 | """""" | |
58 |
|
58 | |||
59 |
|
59 | |||
60 | class IUnSerialized: |
|
60 | class IUnSerialized: | |
61 |
|
61 | |||
62 | def getObject(): |
|
62 | def getObject(): | |
63 | """""" |
|
63 | """""" | |
64 |
|
64 | |||
65 | class Serialized(object): |
|
65 | class Serialized(object): | |
66 |
|
66 | |||
67 | # implements(ISerialized) |
|
67 | # implements(ISerialized) | |
68 |
|
68 | |||
69 | def __init__(self, data, typeDescriptor, metadata={}): |
|
69 | def __init__(self, data, typeDescriptor, metadata={}): | |
70 | self.data = data |
|
70 | self.data = data | |
71 | self.typeDescriptor = typeDescriptor |
|
71 | self.typeDescriptor = typeDescriptor | |
72 | self.metadata = metadata |
|
72 | self.metadata = metadata | |
73 |
|
73 | |||
74 | def getData(self): |
|
74 | def getData(self): | |
75 | return self.data |
|
75 | return self.data | |
76 |
|
76 | |||
77 | def getDataSize(self, units=10.0**6): |
|
77 | def getDataSize(self, units=10.0**6): | |
78 | return len(self.data)/units |
|
78 | return len(self.data)/units | |
79 |
|
79 | |||
80 | def getTypeDescriptor(self): |
|
80 | def getTypeDescriptor(self): | |
81 | return self.typeDescriptor |
|
81 | return self.typeDescriptor | |
82 |
|
82 | |||
83 | def getMetadata(self): |
|
83 | def getMetadata(self): | |
84 | return self.metadata |
|
84 | return self.metadata | |
85 |
|
85 | |||
86 |
|
86 | |||
87 | class UnSerialized(object): |
|
87 | class UnSerialized(object): | |
88 |
|
88 | |||
89 | # implements(IUnSerialized) |
|
89 | # implements(IUnSerialized) | |
90 |
|
90 | |||
91 | def __init__(self, obj): |
|
91 | def __init__(self, obj): | |
92 | self.obj = obj |
|
92 | self.obj = obj | |
93 |
|
93 | |||
94 | def getObject(self): |
|
94 | def getObject(self): | |
95 | return self.obj |
|
95 | return self.obj | |
96 |
|
96 | |||
97 |
|
97 | |||
98 | class SerializeIt(object): |
|
98 | class SerializeIt(object): | |
99 |
|
99 | |||
100 | # implements(ISerialized) |
|
100 | # implements(ISerialized) | |
101 |
|
101 | |||
102 | def __init__(self, unSerialized): |
|
102 | def __init__(self, unSerialized): | |
103 | self.data = None |
|
103 | self.data = None | |
104 | self.obj = unSerialized.getObject() |
|
104 | self.obj = unSerialized.getObject() | |
105 | if numpy is not None and isinstance(self.obj, numpy.ndarray): |
|
105 | if numpy is not None and isinstance(self.obj, numpy.ndarray): | |
106 | if len(self.obj.shape) == 0: # length 0 arrays are just pickled |
|
106 | if len(self.obj.shape) == 0: # length 0 arrays are just pickled | |
107 | self.typeDescriptor = 'pickle' |
|
107 | self.typeDescriptor = 'pickle' | |
108 | self.metadata = {} |
|
108 | self.metadata = {} | |
109 | else: |
|
109 | else: | |
110 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) |
|
110 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) | |
111 | self.typeDescriptor = 'ndarray' |
|
111 | self.typeDescriptor = 'ndarray' | |
112 | self.metadata = {'shape':self.obj.shape, |
|
112 | self.metadata = {'shape':self.obj.shape, | |
113 |
'dtype':self.obj.dtype |
|
113 | 'dtype':self.obj.dtype} | |
114 | elif isinstance(self.obj, bytes): |
|
114 | elif isinstance(self.obj, bytes): | |
115 | self.typeDescriptor = 'bytes' |
|
115 | self.typeDescriptor = 'bytes' | |
116 | self.metadata = {} |
|
116 | self.metadata = {} | |
117 | elif isinstance(self.obj, buffer): |
|
117 | elif isinstance(self.obj, buffer): | |
118 | self.typeDescriptor = 'buffer' |
|
118 | self.typeDescriptor = 'buffer' | |
119 | self.metadata = {} |
|
119 | self.metadata = {} | |
120 | else: |
|
120 | else: | |
121 | self.typeDescriptor = 'pickle' |
|
121 | self.typeDescriptor = 'pickle' | |
122 | self.metadata = {} |
|
122 | self.metadata = {} | |
123 | self._generateData() |
|
123 | self._generateData() | |
124 |
|
124 | |||
125 | def _generateData(self): |
|
125 | def _generateData(self): | |
126 | if self.typeDescriptor == 'ndarray': |
|
126 | if self.typeDescriptor == 'ndarray': | |
127 | self.data = buffer(self.obj) |
|
127 | self.data = buffer(self.obj) | |
128 | elif self.typeDescriptor in ('bytes', 'buffer'): |
|
128 | elif self.typeDescriptor in ('bytes', 'buffer'): | |
129 | self.data = self.obj |
|
129 | self.data = self.obj | |
130 | elif self.typeDescriptor == 'pickle': |
|
130 | elif self.typeDescriptor == 'pickle': | |
131 | self.data = pickle.dumps(self.obj, -1) |
|
131 | self.data = pickle.dumps(self.obj, -1) | |
132 | else: |
|
132 | else: | |
133 | raise SerializationError("Really wierd serialization error.") |
|
133 | raise SerializationError("Really wierd serialization error.") | |
134 | del self.obj |
|
134 | del self.obj | |
135 |
|
135 | |||
136 | def getData(self): |
|
136 | def getData(self): | |
137 | return self.data |
|
137 | return self.data | |
138 |
|
138 | |||
139 | def getDataSize(self, units=10.0**6): |
|
139 | def getDataSize(self, units=10.0**6): | |
140 | return 1.0*len(self.data)/units |
|
140 | return 1.0*len(self.data)/units | |
141 |
|
141 | |||
142 | def getTypeDescriptor(self): |
|
142 | def getTypeDescriptor(self): | |
143 | return self.typeDescriptor |
|
143 | return self.typeDescriptor | |
144 |
|
144 | |||
145 | def getMetadata(self): |
|
145 | def getMetadata(self): | |
146 | return self.metadata |
|
146 | return self.metadata | |
147 |
|
147 | |||
148 |
|
148 | |||
149 | class UnSerializeIt(UnSerialized): |
|
149 | class UnSerializeIt(UnSerialized): | |
150 |
|
150 | |||
151 | # implements(IUnSerialized) |
|
151 | # implements(IUnSerialized) | |
152 |
|
152 | |||
153 | def __init__(self, serialized): |
|
153 | def __init__(self, serialized): | |
154 | self.serialized = serialized |
|
154 | self.serialized = serialized | |
155 |
|
155 | |||
156 | def getObject(self): |
|
156 | def getObject(self): | |
157 | typeDescriptor = self.serialized.getTypeDescriptor() |
|
157 | typeDescriptor = self.serialized.getTypeDescriptor() | |
158 | if numpy is not None and typeDescriptor == 'ndarray': |
|
158 | if numpy is not None and typeDescriptor == 'ndarray': | |
159 | buf = self.serialized.getData() |
|
159 | buf = self.serialized.getData() | |
160 | if isinstance(buf, (bytes, buffer, memoryview)): |
|
160 | if isinstance(buf, (bytes, buffer, memoryview)): | |
161 | result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype']) |
|
161 | result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype']) | |
162 | else: |
|
162 | else: | |
163 | raise TypeError("Expected bytes or buffer/memoryview, but got %r"%type(buf)) |
|
163 | raise TypeError("Expected bytes or buffer/memoryview, but got %r"%type(buf)) | |
164 | result.shape = self.serialized.metadata['shape'] |
|
164 | result.shape = self.serialized.metadata['shape'] | |
165 | elif typeDescriptor == 'pickle': |
|
165 | elif typeDescriptor == 'pickle': | |
166 | result = pickle.loads(self.serialized.getData()) |
|
166 | result = pickle.loads(self.serialized.getData()) | |
167 | elif typeDescriptor in ('bytes', 'buffer'): |
|
167 | elif typeDescriptor in ('bytes', 'buffer'): | |
168 | result = self.serialized.getData() |
|
168 | result = self.serialized.getData() | |
169 | else: |
|
169 | else: | |
170 | raise SerializationError("Really wierd serialization error.") |
|
170 | raise SerializationError("Really wierd serialization error.") | |
171 | return result |
|
171 | return result | |
172 |
|
172 | |||
173 | def serialize(obj): |
|
173 | def serialize(obj): | |
174 | return SerializeIt(UnSerialized(obj)) |
|
174 | return SerializeIt(UnSerialized(obj)) | |
175 |
|
175 | |||
176 | def unserialize(serialized): |
|
176 | def unserialize(serialized): | |
177 | return UnSerializeIt(serialized).getObject() |
|
177 | return UnSerializeIt(serialized).getObject() |
General Comments 0
You need to be logged in to leave comments.
Login now