##// END OF EJS Templates
use math to check for nan/inf
MinRK -
Show More
@@ -1,212 +1,207 b''
1 """Utilities to manipulate JSON objects.
1 """Utilities to manipulate JSON objects.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (C) 2010-2011 The IPython Development Team
4 # Copyright (C) 2010-2011 The IPython Development Team
5 #
5 #
6 # Distributed under the terms of the BSD License. The full license is in
6 # Distributed under the terms of the BSD License. The full license is in
7 # the file COPYING.txt, distributed as part of this software.
7 # the file COPYING.txt, distributed as part of this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # stdlib
13 # stdlib
14 import math
14 import re
15 import re
15 import sys
16 import sys
16 import types
17 import types
17 from base64 import encodestring
18 from base64 import encodestring
18 from datetime import datetime
19 from datetime import datetime
19
20
20 from IPython.utils import py3compat
21 from IPython.utils import py3compat
21 from IPython.utils.encoding import DEFAULT_ENCODING
22 from IPython.utils.encoding import DEFAULT_ENCODING
22 from IPython.utils import text
23 from IPython.utils import text
23 next_attr_name = '__next__' if py3compat.PY3 else 'next'
24 next_attr_name = '__next__' if py3compat.PY3 else 'next'
24
25
25 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
26 # Globals and constants
27 # Globals and constants
27 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
28
29
29 # timestamp formats
30 # timestamp formats
30 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
31 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
31 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
32 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
32
33
33 # float constants
34 NAN = float('nan')
35 INF = float('inf')
36 NINF = float('-inf')
37 INFS = (INF, NINF)
38
39 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
40 # Classes and functions
35 # Classes and functions
41 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
42
37
43 def rekey(dikt):
38 def rekey(dikt):
44 """Rekey a dict that has been forced to use str keys where there should be
39 """Rekey a dict that has been forced to use str keys where there should be
45 ints by json."""
40 ints by json."""
46 for k in dikt.iterkeys():
41 for k in dikt.iterkeys():
47 if isinstance(k, basestring):
42 if isinstance(k, basestring):
48 ik=fk=None
43 ik=fk=None
49 try:
44 try:
50 ik = int(k)
45 ik = int(k)
51 except ValueError:
46 except ValueError:
52 try:
47 try:
53 fk = float(k)
48 fk = float(k)
54 except ValueError:
49 except ValueError:
55 continue
50 continue
56 if ik is not None:
51 if ik is not None:
57 nk = ik
52 nk = ik
58 else:
53 else:
59 nk = fk
54 nk = fk
60 if nk in dikt:
55 if nk in dikt:
61 raise KeyError("already have key %r"%nk)
56 raise KeyError("already have key %r"%nk)
62 dikt[nk] = dikt.pop(k)
57 dikt[nk] = dikt.pop(k)
63 return dikt
58 return dikt
64
59
65
60
66 def extract_dates(obj):
61 def extract_dates(obj):
67 """extract ISO8601 dates from unpacked JSON"""
62 """extract ISO8601 dates from unpacked JSON"""
68 if isinstance(obj, dict):
63 if isinstance(obj, dict):
69 obj = dict(obj) # don't clobber
64 obj = dict(obj) # don't clobber
70 for k,v in obj.iteritems():
65 for k,v in obj.iteritems():
71 obj[k] = extract_dates(v)
66 obj[k] = extract_dates(v)
72 elif isinstance(obj, (list, tuple)):
67 elif isinstance(obj, (list, tuple)):
73 obj = [ extract_dates(o) for o in obj ]
68 obj = [ extract_dates(o) for o in obj ]
74 elif isinstance(obj, basestring):
69 elif isinstance(obj, basestring):
75 if ISO8601_PAT.match(obj):
70 if ISO8601_PAT.match(obj):
76 obj = datetime.strptime(obj, ISO8601)
71 obj = datetime.strptime(obj, ISO8601)
77 return obj
72 return obj
78
73
79 def squash_dates(obj):
74 def squash_dates(obj):
80 """squash datetime objects into ISO8601 strings"""
75 """squash datetime objects into ISO8601 strings"""
81 if isinstance(obj, dict):
76 if isinstance(obj, dict):
82 obj = dict(obj) # don't clobber
77 obj = dict(obj) # don't clobber
83 for k,v in obj.iteritems():
78 for k,v in obj.iteritems():
84 obj[k] = squash_dates(v)
79 obj[k] = squash_dates(v)
85 elif isinstance(obj, (list, tuple)):
80 elif isinstance(obj, (list, tuple)):
86 obj = [ squash_dates(o) for o in obj ]
81 obj = [ squash_dates(o) for o in obj ]
87 elif isinstance(obj, datetime):
82 elif isinstance(obj, datetime):
88 obj = obj.strftime(ISO8601)
83 obj = obj.strftime(ISO8601)
89 return obj
84 return obj
90
85
91 def date_default(obj):
86 def date_default(obj):
92 """default function for packing datetime objects in JSON."""
87 """default function for packing datetime objects in JSON."""
93 if isinstance(obj, datetime):
88 if isinstance(obj, datetime):
94 return obj.strftime(ISO8601)
89 return obj.strftime(ISO8601)
95 else:
90 else:
96 raise TypeError("%r is not JSON serializable"%obj)
91 raise TypeError("%r is not JSON serializable"%obj)
97
92
98
93
99 # constants for identifying png/jpeg data
94 # constants for identifying png/jpeg data
100 PNG = b'\x89PNG\r\n\x1a\n'
95 PNG = b'\x89PNG\r\n\x1a\n'
101 JPEG = b'\xff\xd8'
96 JPEG = b'\xff\xd8'
102
97
103 def encode_images(format_dict):
98 def encode_images(format_dict):
104 """b64-encodes images in a displaypub format dict
99 """b64-encodes images in a displaypub format dict
105
100
106 Perhaps this should be handled in json_clean itself?
101 Perhaps this should be handled in json_clean itself?
107
102
108 Parameters
103 Parameters
109 ----------
104 ----------
110
105
111 format_dict : dict
106 format_dict : dict
112 A dictionary of display data keyed by mime-type
107 A dictionary of display data keyed by mime-type
113
108
114 Returns
109 Returns
115 -------
110 -------
116
111
117 format_dict : dict
112 format_dict : dict
118 A copy of the same dictionary,
113 A copy of the same dictionary,
119 but binary image data ('image/png' or 'image/jpeg')
114 but binary image data ('image/png' or 'image/jpeg')
120 is base64-encoded.
115 is base64-encoded.
121
116
122 """
117 """
123 encoded = format_dict.copy()
118 encoded = format_dict.copy()
124 pngdata = format_dict.get('image/png')
119 pngdata = format_dict.get('image/png')
125 if isinstance(pngdata, bytes) and pngdata[:8] == PNG:
120 if isinstance(pngdata, bytes) and pngdata[:8] == PNG:
126 encoded['image/png'] = encodestring(pngdata).decode('ascii')
121 encoded['image/png'] = encodestring(pngdata).decode('ascii')
127 jpegdata = format_dict.get('image/jpeg')
122 jpegdata = format_dict.get('image/jpeg')
128 if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:
123 if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:
129 encoded['image/jpeg'] = encodestring(jpegdata).decode('ascii')
124 encoded['image/jpeg'] = encodestring(jpegdata).decode('ascii')
130 return encoded
125 return encoded
131
126
132
127
133 def json_clean(obj):
128 def json_clean(obj):
134 """Clean an object to ensure it's safe to encode in JSON.
129 """Clean an object to ensure it's safe to encode in JSON.
135
130
136 Atomic, immutable objects are returned unmodified. Sets and tuples are
131 Atomic, immutable objects are returned unmodified. Sets and tuples are
137 converted to lists, lists are copied and dicts are also copied.
132 converted to lists, lists are copied and dicts are also copied.
138
133
139 Note: dicts whose keys could cause collisions upon encoding (such as a dict
134 Note: dicts whose keys could cause collisions upon encoding (such as a dict
140 with both the number 1 and the string '1' as keys) will cause a ValueError
135 with both the number 1 and the string '1' as keys) will cause a ValueError
141 to be raised.
136 to be raised.
142
137
143 Parameters
138 Parameters
144 ----------
139 ----------
145 obj : any python object
140 obj : any python object
146
141
147 Returns
142 Returns
148 -------
143 -------
149 out : object
144 out : object
150
145
151 A version of the input which will not cause an encoding error when
146 A version of the input which will not cause an encoding error when
152 encoded as JSON. Note that this function does not *encode* its inputs,
147 encoded as JSON. Note that this function does not *encode* its inputs,
153 it simply sanitizes it so that there will be no encoding errors later.
148 it simply sanitizes it so that there will be no encoding errors later.
154
149
155 Examples
150 Examples
156 --------
151 --------
157 >>> json_clean(4)
152 >>> json_clean(4)
158 4
153 4
159 >>> json_clean(range(10))
154 >>> json_clean(range(10))
160 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
155 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
161 >>> sorted(json_clean(dict(x=1, y=2)).items())
156 >>> sorted(json_clean(dict(x=1, y=2)).items())
162 [('x', 1), ('y', 2)]
157 [('x', 1), ('y', 2)]
163 >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
158 >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
164 [('x', 1), ('y', 2), ('z', [1, 2, 3])]
159 [('x', 1), ('y', 2), ('z', [1, 2, 3])]
165 >>> json_clean(True)
160 >>> json_clean(True)
166 True
161 True
167 """
162 """
168 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
163 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
169 # listed explicitly because bools pass as int instances
164 # listed explicitly because bools pass as int instances
170 atomic_ok = (unicode, int, types.NoneType)
165 atomic_ok = (unicode, int, types.NoneType)
171
166
172 # containers that we need to convert into lists
167 # containers that we need to convert into lists
173 container_to_list = (tuple, set, types.GeneratorType)
168 container_to_list = (tuple, set, types.GeneratorType)
174
169
175 if isinstance(obj, float):
170 if isinstance(obj, float):
176 # cast out-of-range floats to their reprs
171 # cast out-of-range floats to their reprs
177 if obj != obj or obj in INFS:
172 if math.isnan(obj) or math.isinf(obj):
178 return repr(obj)
173 return repr(obj)
179 return obj
174 return obj
180
175
181 if isinstance(obj, atomic_ok):
176 if isinstance(obj, atomic_ok):
182 return obj
177 return obj
183
178
184 if isinstance(obj, bytes):
179 if isinstance(obj, bytes):
185 return obj.decode(DEFAULT_ENCODING, 'replace')
180 return obj.decode(DEFAULT_ENCODING, 'replace')
186
181
187 if isinstance(obj, container_to_list) or (
182 if isinstance(obj, container_to_list) or (
188 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
183 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
189 obj = list(obj)
184 obj = list(obj)
190
185
191 if isinstance(obj, list):
186 if isinstance(obj, list):
192 return [json_clean(x) for x in obj]
187 return [json_clean(x) for x in obj]
193
188
194 if isinstance(obj, dict):
189 if isinstance(obj, dict):
195 # First, validate that the dict won't lose data in conversion due to
190 # First, validate that the dict won't lose data in conversion due to
196 # key collisions after stringification. This can happen with keys like
191 # key collisions after stringification. This can happen with keys like
197 # True and 'true' or 1 and '1', which collide in JSON.
192 # True and 'true' or 1 and '1', which collide in JSON.
198 nkeys = len(obj)
193 nkeys = len(obj)
199 nkeys_collapsed = len(set(map(str, obj)))
194 nkeys_collapsed = len(set(map(str, obj)))
200 if nkeys != nkeys_collapsed:
195 if nkeys != nkeys_collapsed:
201 raise ValueError('dict can not be safely converted to JSON: '
196 raise ValueError('dict can not be safely converted to JSON: '
202 'key collision would lead to dropped values')
197 'key collision would lead to dropped values')
203 # If all OK, proceed by making the new dict that will be json-safe
198 # If all OK, proceed by making the new dict that will be json-safe
204 out = {}
199 out = {}
205 for k,v in obj.iteritems():
200 for k,v in obj.iteritems():
206 out[str(k)] = json_clean(v)
201 out[str(k)] = json_clean(v)
207 return out
202 return out
208
203
209 # If we get here, we don't know how to handle the object, so we just get
204 # If we get here, we don't know how to handle the object, so we just get
210 # its repr and return that. This will catch lambdas, open sockets, class
205 # its repr and return that. This will catch lambdas, open sockets, class
211 # objects, and any other complicated contraption that json can't encode
206 # objects, and any other complicated contraption that json can't encode
212 return repr(obj)
207 return repr(obj)
General Comments 0
You need to be logged in to leave comments. Login now