##// END OF EJS Templates
P3K: fix DeprecationWarning under Python 3.x (base64.encodestring is deprecated)
Mikhail Korobov -
Show More
@@ -1,207 +1,213 b''
1 """Utilities to manipulate JSON objects.
1 """Utilities to manipulate JSON objects.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (C) 2010-2011 The IPython Development Team
4 # Copyright (C) 2010-2011 The IPython Development Team
5 #
5 #
6 # Distributed under the terms of the BSD License. The full license is in
6 # Distributed under the terms of the BSD License. The full license is in
7 # the file COPYING.txt, distributed as part of this software.
7 # the file COPYING.txt, distributed as part of this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # stdlib
13 # stdlib
14 import math
14 import math
15 import re
15 import re
16 import sys
16 import sys
17 import types
17 import types
18 from base64 import encodestring
19 from datetime import datetime
18 from datetime import datetime
20
19
20 try:
21 # base64.encodestring is deprecated in Python 3.x
22 from base64 import encodebytes
23 except ImportError:
24 # Python 2.x
25 from base64 import encodestring as encodebytes
26
21 from IPython.utils import py3compat
27 from IPython.utils import py3compat
22 from IPython.utils.encoding import DEFAULT_ENCODING
28 from IPython.utils.encoding import DEFAULT_ENCODING
23 from IPython.utils import text
29 from IPython.utils import text
24 next_attr_name = '__next__' if py3compat.PY3 else 'next'
30 next_attr_name = '__next__' if py3compat.PY3 else 'next'
25
31
26 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
27 # Globals and constants
33 # Globals and constants
28 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
29
35
30 # timestamp formats
36 # timestamp formats
31 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
37 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
32 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
38 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
33
39
34 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
35 # Classes and functions
41 # Classes and functions
36 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
37
43
38 def rekey(dikt):
44 def rekey(dikt):
39 """Rekey a dict that has been forced to use str keys where there should be
45 """Rekey a dict that has been forced to use str keys where there should be
40 ints by json."""
46 ints by json."""
41 for k in dikt.iterkeys():
47 for k in dikt.iterkeys():
42 if isinstance(k, basestring):
48 if isinstance(k, basestring):
43 ik=fk=None
49 ik=fk=None
44 try:
50 try:
45 ik = int(k)
51 ik = int(k)
46 except ValueError:
52 except ValueError:
47 try:
53 try:
48 fk = float(k)
54 fk = float(k)
49 except ValueError:
55 except ValueError:
50 continue
56 continue
51 if ik is not None:
57 if ik is not None:
52 nk = ik
58 nk = ik
53 else:
59 else:
54 nk = fk
60 nk = fk
55 if nk in dikt:
61 if nk in dikt:
56 raise KeyError("already have key %r"%nk)
62 raise KeyError("already have key %r"%nk)
57 dikt[nk] = dikt.pop(k)
63 dikt[nk] = dikt.pop(k)
58 return dikt
64 return dikt
59
65
60
66
61 def extract_dates(obj):
67 def extract_dates(obj):
62 """extract ISO8601 dates from unpacked JSON"""
68 """extract ISO8601 dates from unpacked JSON"""
63 if isinstance(obj, dict):
69 if isinstance(obj, dict):
64 obj = dict(obj) # don't clobber
70 obj = dict(obj) # don't clobber
65 for k,v in obj.iteritems():
71 for k,v in obj.iteritems():
66 obj[k] = extract_dates(v)
72 obj[k] = extract_dates(v)
67 elif isinstance(obj, (list, tuple)):
73 elif isinstance(obj, (list, tuple)):
68 obj = [ extract_dates(o) for o in obj ]
74 obj = [ extract_dates(o) for o in obj ]
69 elif isinstance(obj, basestring):
75 elif isinstance(obj, basestring):
70 if ISO8601_PAT.match(obj):
76 if ISO8601_PAT.match(obj):
71 obj = datetime.strptime(obj, ISO8601)
77 obj = datetime.strptime(obj, ISO8601)
72 return obj
78 return obj
73
79
74 def squash_dates(obj):
80 def squash_dates(obj):
75 """squash datetime objects into ISO8601 strings"""
81 """squash datetime objects into ISO8601 strings"""
76 if isinstance(obj, dict):
82 if isinstance(obj, dict):
77 obj = dict(obj) # don't clobber
83 obj = dict(obj) # don't clobber
78 for k,v in obj.iteritems():
84 for k,v in obj.iteritems():
79 obj[k] = squash_dates(v)
85 obj[k] = squash_dates(v)
80 elif isinstance(obj, (list, tuple)):
86 elif isinstance(obj, (list, tuple)):
81 obj = [ squash_dates(o) for o in obj ]
87 obj = [ squash_dates(o) for o in obj ]
82 elif isinstance(obj, datetime):
88 elif isinstance(obj, datetime):
83 obj = obj.strftime(ISO8601)
89 obj = obj.strftime(ISO8601)
84 return obj
90 return obj
85
91
86 def date_default(obj):
92 def date_default(obj):
87 """default function for packing datetime objects in JSON."""
93 """default function for packing datetime objects in JSON."""
88 if isinstance(obj, datetime):
94 if isinstance(obj, datetime):
89 return obj.strftime(ISO8601)
95 return obj.strftime(ISO8601)
90 else:
96 else:
91 raise TypeError("%r is not JSON serializable"%obj)
97 raise TypeError("%r is not JSON serializable"%obj)
92
98
93
99
94 # constants for identifying png/jpeg data
100 # constants for identifying png/jpeg data
95 PNG = b'\x89PNG\r\n\x1a\n'
101 PNG = b'\x89PNG\r\n\x1a\n'
96 JPEG = b'\xff\xd8'
102 JPEG = b'\xff\xd8'
97
103
98 def encode_images(format_dict):
104 def encode_images(format_dict):
99 """b64-encodes images in a displaypub format dict
105 """b64-encodes images in a displaypub format dict
100
106
101 Perhaps this should be handled in json_clean itself?
107 Perhaps this should be handled in json_clean itself?
102
108
103 Parameters
109 Parameters
104 ----------
110 ----------
105
111
106 format_dict : dict
112 format_dict : dict
107 A dictionary of display data keyed by mime-type
113 A dictionary of display data keyed by mime-type
108
114
109 Returns
115 Returns
110 -------
116 -------
111
117
112 format_dict : dict
118 format_dict : dict
113 A copy of the same dictionary,
119 A copy of the same dictionary,
114 but binary image data ('image/png' or 'image/jpeg')
120 but binary image data ('image/png' or 'image/jpeg')
115 is base64-encoded.
121 is base64-encoded.
116
122
117 """
123 """
118 encoded = format_dict.copy()
124 encoded = format_dict.copy()
119 pngdata = format_dict.get('image/png')
125 pngdata = format_dict.get('image/png')
120 if isinstance(pngdata, bytes) and pngdata[:8] == PNG:
126 if isinstance(pngdata, bytes) and pngdata[:8] == PNG:
121 encoded['image/png'] = encodestring(pngdata).decode('ascii')
127 encoded['image/png'] = encodebytes(pngdata).decode('ascii')
122 jpegdata = format_dict.get('image/jpeg')
128 jpegdata = format_dict.get('image/jpeg')
123 if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:
129 if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:
124 encoded['image/jpeg'] = encodestring(jpegdata).decode('ascii')
130 encoded['image/jpeg'] = encodebytes(jpegdata).decode('ascii')
125 return encoded
131 return encoded
126
132
127
133
128 def json_clean(obj):
134 def json_clean(obj):
129 """Clean an object to ensure it's safe to encode in JSON.
135 """Clean an object to ensure it's safe to encode in JSON.
130
136
131 Atomic, immutable objects are returned unmodified. Sets and tuples are
137 Atomic, immutable objects are returned unmodified. Sets and tuples are
132 converted to lists, lists are copied and dicts are also copied.
138 converted to lists, lists are copied and dicts are also copied.
133
139
134 Note: dicts whose keys could cause collisions upon encoding (such as a dict
140 Note: dicts whose keys could cause collisions upon encoding (such as a dict
135 with both the number 1 and the string '1' as keys) will cause a ValueError
141 with both the number 1 and the string '1' as keys) will cause a ValueError
136 to be raised.
142 to be raised.
137
143
138 Parameters
144 Parameters
139 ----------
145 ----------
140 obj : any python object
146 obj : any python object
141
147
142 Returns
148 Returns
143 -------
149 -------
144 out : object
150 out : object
145
151
146 A version of the input which will not cause an encoding error when
152 A version of the input which will not cause an encoding error when
147 encoded as JSON. Note that this function does not *encode* its inputs,
153 encoded as JSON. Note that this function does not *encode* its inputs,
148 it simply sanitizes it so that there will be no encoding errors later.
154 it simply sanitizes it so that there will be no encoding errors later.
149
155
150 Examples
156 Examples
151 --------
157 --------
152 >>> json_clean(4)
158 >>> json_clean(4)
153 4
159 4
154 >>> json_clean(range(10))
160 >>> json_clean(range(10))
155 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
161 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
156 >>> sorted(json_clean(dict(x=1, y=2)).items())
162 >>> sorted(json_clean(dict(x=1, y=2)).items())
157 [('x', 1), ('y', 2)]
163 [('x', 1), ('y', 2)]
158 >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
164 >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
159 [('x', 1), ('y', 2), ('z', [1, 2, 3])]
165 [('x', 1), ('y', 2), ('z', [1, 2, 3])]
160 >>> json_clean(True)
166 >>> json_clean(True)
161 True
167 True
162 """
168 """
163 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
169 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
164 # listed explicitly because bools pass as int instances
170 # listed explicitly because bools pass as int instances
165 atomic_ok = (unicode, int, types.NoneType)
171 atomic_ok = (unicode, int, types.NoneType)
166
172
167 # containers that we need to convert into lists
173 # containers that we need to convert into lists
168 container_to_list = (tuple, set, types.GeneratorType)
174 container_to_list = (tuple, set, types.GeneratorType)
169
175
170 if isinstance(obj, float):
176 if isinstance(obj, float):
171 # cast out-of-range floats to their reprs
177 # cast out-of-range floats to their reprs
172 if math.isnan(obj) or math.isinf(obj):
178 if math.isnan(obj) or math.isinf(obj):
173 return repr(obj)
179 return repr(obj)
174 return obj
180 return obj
175
181
176 if isinstance(obj, atomic_ok):
182 if isinstance(obj, atomic_ok):
177 return obj
183 return obj
178
184
179 if isinstance(obj, bytes):
185 if isinstance(obj, bytes):
180 return obj.decode(DEFAULT_ENCODING, 'replace')
186 return obj.decode(DEFAULT_ENCODING, 'replace')
181
187
182 if isinstance(obj, container_to_list) or (
188 if isinstance(obj, container_to_list) or (
183 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
189 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
184 obj = list(obj)
190 obj = list(obj)
185
191
186 if isinstance(obj, list):
192 if isinstance(obj, list):
187 return [json_clean(x) for x in obj]
193 return [json_clean(x) for x in obj]
188
194
189 if isinstance(obj, dict):
195 if isinstance(obj, dict):
190 # First, validate that the dict won't lose data in conversion due to
196 # First, validate that the dict won't lose data in conversion due to
191 # key collisions after stringification. This can happen with keys like
197 # key collisions after stringification. This can happen with keys like
192 # True and 'true' or 1 and '1', which collide in JSON.
198 # True and 'true' or 1 and '1', which collide in JSON.
193 nkeys = len(obj)
199 nkeys = len(obj)
194 nkeys_collapsed = len(set(map(str, obj)))
200 nkeys_collapsed = len(set(map(str, obj)))
195 if nkeys != nkeys_collapsed:
201 if nkeys != nkeys_collapsed:
196 raise ValueError('dict can not be safely converted to JSON: '
202 raise ValueError('dict can not be safely converted to JSON: '
197 'key collision would lead to dropped values')
203 'key collision would lead to dropped values')
198 # If all OK, proceed by making the new dict that will be json-safe
204 # If all OK, proceed by making the new dict that will be json-safe
199 out = {}
205 out = {}
200 for k,v in obj.iteritems():
206 for k,v in obj.iteritems():
201 out[str(k)] = json_clean(v)
207 out[str(k)] = json_clean(v)
202 return out
208 return out
203
209
204 # If we get here, we don't know how to handle the object, so we just get
210 # If we get here, we don't know how to handle the object, so we just get
205 # its repr and return that. This will catch lambdas, open sockets, class
211 # its repr and return that. This will catch lambdas, open sockets, class
206 # objects, and any other complicated contraption that json can't encode
212 # objects, and any other complicated contraption that json can't encode
207 return repr(obj)
213 return repr(obj)
General Comments 0
You need to be logged in to leave comments. Login now