##// END OF EJS Templates
include dtype itself in metadata, not str representation...
MinRK -
Show More
@@ -1,177 +1,177 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 # -*- test-case-name: IPython.kernel.test.test_newserialized -*-
2 # -*- test-case-name: IPython.kernel.test.test_newserialized -*-
3
3
4 """Refactored serialization classes and interfaces."""
4 """Refactored serialization classes and interfaces."""
5
5
6 __docformat__ = "restructuredtext en"
6 __docformat__ = "restructuredtext en"
7
7
8 # Tell nose to skip this module
8 # Tell nose to skip this module
9 __test__ = {}
9 __test__ = {}
10
10
11 #-------------------------------------------------------------------------------
11 #-------------------------------------------------------------------------------
12 # Copyright (C) 2008-2011 The IPython Development Team
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-------------------------------------------------------------------------------
16 #-------------------------------------------------------------------------------
17
17
18 #-------------------------------------------------------------------------------
18 #-------------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-------------------------------------------------------------------------------
20 #-------------------------------------------------------------------------------
21
21
22 import sys
22 import sys
23 import cPickle as pickle
23 import cPickle as pickle
24
24
25 try:
25 try:
26 import numpy
26 import numpy
27 except ImportError:
27 except ImportError:
28 numpy = None
28 numpy = None
29
29
30 class SerializationError(Exception):
30 class SerializationError(Exception):
31 pass
31 pass
32
32
33 if sys.version_info[0] >= 3:
33 if sys.version_info[0] >= 3:
34 buffer = memoryview
34 buffer = memoryview
35 py3k = True
35 py3k = True
36 else:
36 else:
37 py3k = False
37 py3k = False
38 if sys.version_info[:2] <= (2,6):
38 if sys.version_info[:2] <= (2,6):
39 memoryview = buffer
39 memoryview = buffer
40
40
41 #-----------------------------------------------------------------------------
41 #-----------------------------------------------------------------------------
42 # Classes and functions
42 # Classes and functions
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44
44
45 class ISerialized:
45 class ISerialized:
46
46
47 def getData():
47 def getData():
48 """"""
48 """"""
49
49
50 def getDataSize(units=10.0**6):
50 def getDataSize(units=10.0**6):
51 """"""
51 """"""
52
52
53 def getTypeDescriptor():
53 def getTypeDescriptor():
54 """"""
54 """"""
55
55
56 def getMetadata():
56 def getMetadata():
57 """"""
57 """"""
58
58
59
59
60 class IUnSerialized:
60 class IUnSerialized:
61
61
62 def getObject():
62 def getObject():
63 """"""
63 """"""
64
64
65 class Serialized(object):
65 class Serialized(object):
66
66
67 # implements(ISerialized)
67 # implements(ISerialized)
68
68
69 def __init__(self, data, typeDescriptor, metadata={}):
69 def __init__(self, data, typeDescriptor, metadata={}):
70 self.data = data
70 self.data = data
71 self.typeDescriptor = typeDescriptor
71 self.typeDescriptor = typeDescriptor
72 self.metadata = metadata
72 self.metadata = metadata
73
73
74 def getData(self):
74 def getData(self):
75 return self.data
75 return self.data
76
76
77 def getDataSize(self, units=10.0**6):
77 def getDataSize(self, units=10.0**6):
78 return len(self.data)/units
78 return len(self.data)/units
79
79
80 def getTypeDescriptor(self):
80 def getTypeDescriptor(self):
81 return self.typeDescriptor
81 return self.typeDescriptor
82
82
83 def getMetadata(self):
83 def getMetadata(self):
84 return self.metadata
84 return self.metadata
85
85
86
86
87 class UnSerialized(object):
87 class UnSerialized(object):
88
88
89 # implements(IUnSerialized)
89 # implements(IUnSerialized)
90
90
91 def __init__(self, obj):
91 def __init__(self, obj):
92 self.obj = obj
92 self.obj = obj
93
93
94 def getObject(self):
94 def getObject(self):
95 return self.obj
95 return self.obj
96
96
97
97
98 class SerializeIt(object):
98 class SerializeIt(object):
99
99
100 # implements(ISerialized)
100 # implements(ISerialized)
101
101
102 def __init__(self, unSerialized):
102 def __init__(self, unSerialized):
103 self.data = None
103 self.data = None
104 self.obj = unSerialized.getObject()
104 self.obj = unSerialized.getObject()
105 if numpy is not None and isinstance(self.obj, numpy.ndarray):
105 if numpy is not None and isinstance(self.obj, numpy.ndarray):
106 if len(self.obj.shape) == 0: # length 0 arrays are just pickled
106 if len(self.obj.shape) == 0: # length 0 arrays are just pickled
107 self.typeDescriptor = 'pickle'
107 self.typeDescriptor = 'pickle'
108 self.metadata = {}
108 self.metadata = {}
109 else:
109 else:
110 self.obj = numpy.ascontiguousarray(self.obj, dtype=None)
110 self.obj = numpy.ascontiguousarray(self.obj, dtype=None)
111 self.typeDescriptor = 'ndarray'
111 self.typeDescriptor = 'ndarray'
112 self.metadata = {'shape':self.obj.shape,
112 self.metadata = {'shape':self.obj.shape,
113 'dtype':self.obj.dtype.str}
113 'dtype':self.obj.dtype}
114 elif isinstance(self.obj, bytes):
114 elif isinstance(self.obj, bytes):
115 self.typeDescriptor = 'bytes'
115 self.typeDescriptor = 'bytes'
116 self.metadata = {}
116 self.metadata = {}
117 elif isinstance(self.obj, buffer):
117 elif isinstance(self.obj, buffer):
118 self.typeDescriptor = 'buffer'
118 self.typeDescriptor = 'buffer'
119 self.metadata = {}
119 self.metadata = {}
120 else:
120 else:
121 self.typeDescriptor = 'pickle'
121 self.typeDescriptor = 'pickle'
122 self.metadata = {}
122 self.metadata = {}
123 self._generateData()
123 self._generateData()
124
124
125 def _generateData(self):
125 def _generateData(self):
126 if self.typeDescriptor == 'ndarray':
126 if self.typeDescriptor == 'ndarray':
127 self.data = buffer(self.obj)
127 self.data = buffer(self.obj)
128 elif self.typeDescriptor in ('bytes', 'buffer'):
128 elif self.typeDescriptor in ('bytes', 'buffer'):
129 self.data = self.obj
129 self.data = self.obj
130 elif self.typeDescriptor == 'pickle':
130 elif self.typeDescriptor == 'pickle':
131 self.data = pickle.dumps(self.obj, -1)
131 self.data = pickle.dumps(self.obj, -1)
132 else:
132 else:
133 raise SerializationError("Really wierd serialization error.")
133 raise SerializationError("Really wierd serialization error.")
134 del self.obj
134 del self.obj
135
135
136 def getData(self):
136 def getData(self):
137 return self.data
137 return self.data
138
138
139 def getDataSize(self, units=10.0**6):
139 def getDataSize(self, units=10.0**6):
140 return 1.0*len(self.data)/units
140 return 1.0*len(self.data)/units
141
141
142 def getTypeDescriptor(self):
142 def getTypeDescriptor(self):
143 return self.typeDescriptor
143 return self.typeDescriptor
144
144
145 def getMetadata(self):
145 def getMetadata(self):
146 return self.metadata
146 return self.metadata
147
147
148
148
149 class UnSerializeIt(UnSerialized):
149 class UnSerializeIt(UnSerialized):
150
150
151 # implements(IUnSerialized)
151 # implements(IUnSerialized)
152
152
153 def __init__(self, serialized):
153 def __init__(self, serialized):
154 self.serialized = serialized
154 self.serialized = serialized
155
155
156 def getObject(self):
156 def getObject(self):
157 typeDescriptor = self.serialized.getTypeDescriptor()
157 typeDescriptor = self.serialized.getTypeDescriptor()
158 if numpy is not None and typeDescriptor == 'ndarray':
158 if numpy is not None and typeDescriptor == 'ndarray':
159 buf = self.serialized.getData()
159 buf = self.serialized.getData()
160 if isinstance(buf, (bytes, buffer, memoryview)):
160 if isinstance(buf, (bytes, buffer, memoryview)):
161 result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype'])
161 result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype'])
162 else:
162 else:
163 raise TypeError("Expected bytes or buffer/memoryview, but got %r"%type(buf))
163 raise TypeError("Expected bytes or buffer/memoryview, but got %r"%type(buf))
164 result.shape = self.serialized.metadata['shape']
164 result.shape = self.serialized.metadata['shape']
165 elif typeDescriptor == 'pickle':
165 elif typeDescriptor == 'pickle':
166 result = pickle.loads(self.serialized.getData())
166 result = pickle.loads(self.serialized.getData())
167 elif typeDescriptor in ('bytes', 'buffer'):
167 elif typeDescriptor in ('bytes', 'buffer'):
168 result = self.serialized.getData()
168 result = self.serialized.getData()
169 else:
169 else:
170 raise SerializationError("Really wierd serialization error.")
170 raise SerializationError("Really wierd serialization error.")
171 return result
171 return result
172
172
173 def serialize(obj):
173 def serialize(obj):
174 return SerializeIt(UnSerialized(obj))
174 return SerializeIt(UnSerialized(obj))
175
175
176 def unserialize(serialized):
176 def unserialize(serialized):
177 return UnSerializeIt(serialized).getObject()
177 return UnSerializeIt(serialized).getObject()
General Comments 0
You need to be logged in to leave comments. Login now