##// END OF EJS Templates
include dtype itself in metadata, not str representation...
MinRK -
Show More
@@ -1,177 +1,177 b''
1 1 # encoding: utf-8
2 2 # -*- test-case-name: IPython.kernel.test.test_newserialized -*-
3 3
4 4 """Refactored serialization classes and interfaces."""
5 5
6 6 __docformat__ = "restructuredtext en"
7 7
8 8 # Tell nose to skip this module
9 9 __test__ = {}
10 10
11 11 #-------------------------------------------------------------------------------
12 12 # Copyright (C) 2008-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-------------------------------------------------------------------------------
17 17
18 18 #-------------------------------------------------------------------------------
19 19 # Imports
20 20 #-------------------------------------------------------------------------------
21 21
22 22 import sys
23 23 import cPickle as pickle
24 24
25 25 try:
26 26 import numpy
27 27 except ImportError:
28 28 numpy = None
29 29
30 30 class SerializationError(Exception):
31 31 pass
32 32
33 33 if sys.version_info[0] >= 3:
34 34 buffer = memoryview
35 35 py3k = True
36 36 else:
37 37 py3k = False
38 38 if sys.version_info[:2] <= (2,6):
39 39 memoryview = buffer
40 40
41 41 #-----------------------------------------------------------------------------
42 42 # Classes and functions
43 43 #-----------------------------------------------------------------------------
44 44
45 45 class ISerialized:
46 46
47 47 def getData():
48 48 """"""
49 49
50 50 def getDataSize(units=10.0**6):
51 51 """"""
52 52
53 53 def getTypeDescriptor():
54 54 """"""
55 55
56 56 def getMetadata():
57 57 """"""
58 58
59 59
60 60 class IUnSerialized:
61 61
62 62 def getObject():
63 63 """"""
64 64
65 65 class Serialized(object):
66 66
67 67 # implements(ISerialized)
68 68
69 69 def __init__(self, data, typeDescriptor, metadata={}):
70 70 self.data = data
71 71 self.typeDescriptor = typeDescriptor
72 72 self.metadata = metadata
73 73
74 74 def getData(self):
75 75 return self.data
76 76
77 77 def getDataSize(self, units=10.0**6):
78 78 return len(self.data)/units
79 79
80 80 def getTypeDescriptor(self):
81 81 return self.typeDescriptor
82 82
83 83 def getMetadata(self):
84 84 return self.metadata
85 85
86 86
87 87 class UnSerialized(object):
88 88
89 89 # implements(IUnSerialized)
90 90
91 91 def __init__(self, obj):
92 92 self.obj = obj
93 93
94 94 def getObject(self):
95 95 return self.obj
96 96
97 97
98 98 class SerializeIt(object):
99 99
100 100 # implements(ISerialized)
101 101
102 102 def __init__(self, unSerialized):
103 103 self.data = None
104 104 self.obj = unSerialized.getObject()
105 105 if numpy is not None and isinstance(self.obj, numpy.ndarray):
106 106 if len(self.obj.shape) == 0: # length 0 arrays are just pickled
107 107 self.typeDescriptor = 'pickle'
108 108 self.metadata = {}
109 109 else:
110 110 self.obj = numpy.ascontiguousarray(self.obj, dtype=None)
111 111 self.typeDescriptor = 'ndarray'
112 112 self.metadata = {'shape':self.obj.shape,
113 'dtype':self.obj.dtype.str}
113 'dtype':self.obj.dtype}
114 114 elif isinstance(self.obj, bytes):
115 115 self.typeDescriptor = 'bytes'
116 116 self.metadata = {}
117 117 elif isinstance(self.obj, buffer):
118 118 self.typeDescriptor = 'buffer'
119 119 self.metadata = {}
120 120 else:
121 121 self.typeDescriptor = 'pickle'
122 122 self.metadata = {}
123 123 self._generateData()
124 124
125 125 def _generateData(self):
126 126 if self.typeDescriptor == 'ndarray':
127 127 self.data = buffer(self.obj)
128 128 elif self.typeDescriptor in ('bytes', 'buffer'):
129 129 self.data = self.obj
130 130 elif self.typeDescriptor == 'pickle':
131 131 self.data = pickle.dumps(self.obj, -1)
132 132 else:
133 133 raise SerializationError("Really wierd serialization error.")
134 134 del self.obj
135 135
136 136 def getData(self):
137 137 return self.data
138 138
139 139 def getDataSize(self, units=10.0**6):
140 140 return 1.0*len(self.data)/units
141 141
142 142 def getTypeDescriptor(self):
143 143 return self.typeDescriptor
144 144
145 145 def getMetadata(self):
146 146 return self.metadata
147 147
148 148
149 149 class UnSerializeIt(UnSerialized):
150 150
151 151 # implements(IUnSerialized)
152 152
153 153 def __init__(self, serialized):
154 154 self.serialized = serialized
155 155
156 156 def getObject(self):
157 157 typeDescriptor = self.serialized.getTypeDescriptor()
158 158 if numpy is not None and typeDescriptor == 'ndarray':
159 159 buf = self.serialized.getData()
160 160 if isinstance(buf, (bytes, buffer, memoryview)):
161 161 result = numpy.frombuffer(buf, dtype = self.serialized.metadata['dtype'])
162 162 else:
163 163 raise TypeError("Expected bytes or buffer/memoryview, but got %r"%type(buf))
164 164 result.shape = self.serialized.metadata['shape']
165 165 elif typeDescriptor == 'pickle':
166 166 result = pickle.loads(self.serialized.getData())
167 167 elif typeDescriptor in ('bytes', 'buffer'):
168 168 result = self.serialized.getData()
169 169 else:
170 170 raise SerializationError("Really wierd serialization error.")
171 171 return result
172 172
173 173 def serialize(obj):
174 174 return SerializeIt(UnSerialized(obj))
175 175
176 176 def unserialize(serialized):
177 177 return UnSerializeIt(serialized).getObject()
General Comments 0
You need to be logged in to leave comments. Login now