##// END OF EJS Templates
don't clobber existing dicts in extract_dates/squash_dates
MinRK -
Show More
@@ -1,132 +1,134 b''
1 """Utilities to manipulate JSON objects.
1 """Utilities to manipulate JSON objects.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (C) 2010 The IPython Development Team
4 # Copyright (C) 2010 The IPython Development Team
5 #
5 #
6 # Distributed under the terms of the BSD License. The full license is in
6 # Distributed under the terms of the BSD License. The full license is in
7 # the file COPYING.txt, distributed as part of this software.
7 # the file COPYING.txt, distributed as part of this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # stdlib
13 # stdlib
14 import re
14 import re
15 import types
15 import types
16 from datetime import datetime
16 from datetime import datetime
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Globals and constants
19 # Globals and constants
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 # timestamp formats
22 # timestamp formats
23 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
23 ISO8601="%Y-%m-%dT%H:%M:%S.%f"
24 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
24 ISO8601_PAT=re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+$")
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Classes and functions
27 # Classes and functions
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29
29
30 def extract_dates(obj):
30 def extract_dates(obj):
31 """extract ISO8601 dates from unpacked JSON"""
31 """extract ISO8601 dates from unpacked JSON"""
32 if isinstance(obj, dict):
32 if isinstance(obj, dict):
33 obj = dict(obj) # don't clobber
33 for k,v in obj.iteritems():
34 for k,v in obj.iteritems():
34 obj[k] = extract_dates(v)
35 obj[k] = extract_dates(v)
35 elif isinstance(obj, (list, tuple)):
36 elif isinstance(obj, (list, tuple)):
36 obj = [ extract_dates(o) for o in obj ]
37 obj = [ extract_dates(o) for o in obj ]
37 elif isinstance(obj, basestring):
38 elif isinstance(obj, basestring):
38 if ISO8601_PAT.match(obj):
39 if ISO8601_PAT.match(obj):
39 obj = datetime.strptime(obj, ISO8601)
40 obj = datetime.strptime(obj, ISO8601)
40 return obj
41 return obj
41
42
42 def squash_dates(obj):
43 def squash_dates(obj):
43 """squash datetime objects into ISO8601 strings"""
44 """squash datetime objects into ISO8601 strings"""
44 if isinstance(obj, dict):
45 if isinstance(obj, dict):
46 obj = dict(obj) # don't clobber
45 for k,v in obj.iteritems():
47 for k,v in obj.iteritems():
46 obj[k] = squash_dates(v)
48 obj[k] = squash_dates(v)
47 elif isinstance(obj, (list, tuple)):
49 elif isinstance(obj, (list, tuple)):
48 obj = [ squash_dates(o) for o in obj ]
50 obj = [ squash_dates(o) for o in obj ]
49 elif isinstance(obj, datetime):
51 elif isinstance(obj, datetime):
50 obj = obj.strftime(ISO8601)
52 obj = obj.strftime(ISO8601)
51 return obj
53 return obj
52
54
53 def date_default(obj):
55 def date_default(obj):
54 """default function for packing datetime objects in JSON."""
56 """default function for packing datetime objects in JSON."""
55 if isinstance(obj, datetime):
57 if isinstance(obj, datetime):
56 return obj.strftime(ISO8601)
58 return obj.strftime(ISO8601)
57 else:
59 else:
58 raise TypeError("%r is not JSON serializable"%obj)
60 raise TypeError("%r is not JSON serializable"%obj)
59
61
60
62
61
63
62 def json_clean(obj):
64 def json_clean(obj):
63 """Clean an object to ensure it's safe to encode in JSON.
65 """Clean an object to ensure it's safe to encode in JSON.
64
66
65 Atomic, immutable objects are returned unmodified. Sets and tuples are
67 Atomic, immutable objects are returned unmodified. Sets and tuples are
66 converted to lists, lists are copied and dicts are also copied.
68 converted to lists, lists are copied and dicts are also copied.
67
69
68 Note: dicts whose keys could cause collisions upon encoding (such as a dict
70 Note: dicts whose keys could cause collisions upon encoding (such as a dict
69 with both the number 1 and the string '1' as keys) will cause a ValueError
71 with both the number 1 and the string '1' as keys) will cause a ValueError
70 to be raised.
72 to be raised.
71
73
72 Parameters
74 Parameters
73 ----------
75 ----------
74 obj : any python object
76 obj : any python object
75
77
76 Returns
78 Returns
77 -------
79 -------
78 out : object
80 out : object
79
81
80 A version of the input which will not cause an encoding error when
82 A version of the input which will not cause an encoding error when
81 encoded as JSON. Note that this function does not *encode* its inputs,
83 encoded as JSON. Note that this function does not *encode* its inputs,
82 it simply sanitizes it so that there will be no encoding errors later.
84 it simply sanitizes it so that there will be no encoding errors later.
83
85
84 Examples
86 Examples
85 --------
87 --------
86 >>> json_clean(4)
88 >>> json_clean(4)
87 4
89 4
88 >>> json_clean(range(10))
90 >>> json_clean(range(10))
89 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
91 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
90 >>> json_clean(dict(x=1, y=2))
92 >>> json_clean(dict(x=1, y=2))
91 {'y': 2, 'x': 1}
93 {'y': 2, 'x': 1}
92 >>> json_clean(dict(x=1, y=2, z=[1,2,3]))
94 >>> json_clean(dict(x=1, y=2, z=[1,2,3]))
93 {'y': 2, 'x': 1, 'z': [1, 2, 3]}
95 {'y': 2, 'x': 1, 'z': [1, 2, 3]}
94 >>> json_clean(True)
96 >>> json_clean(True)
95 True
97 True
96 """
98 """
97 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
99 # types that are 'atomic' and ok in json as-is. bool doesn't need to be
98 # listed explicitly because bools pass as int instances
100 # listed explicitly because bools pass as int instances
99 atomic_ok = (basestring, int, float, types.NoneType)
101 atomic_ok = (basestring, int, float, types.NoneType)
100
102
101 # containers that we need to convert into lists
103 # containers that we need to convert into lists
102 container_to_list = (tuple, set, types.GeneratorType)
104 container_to_list = (tuple, set, types.GeneratorType)
103
105
104 if isinstance(obj, atomic_ok):
106 if isinstance(obj, atomic_ok):
105 return obj
107 return obj
106
108
107 if isinstance(obj, container_to_list) or (
109 if isinstance(obj, container_to_list) or (
108 hasattr(obj, '__iter__') and hasattr(obj, 'next')):
110 hasattr(obj, '__iter__') and hasattr(obj, 'next')):
109 obj = list(obj)
111 obj = list(obj)
110
112
111 if isinstance(obj, list):
113 if isinstance(obj, list):
112 return [json_clean(x) for x in obj]
114 return [json_clean(x) for x in obj]
113
115
114 if isinstance(obj, dict):
116 if isinstance(obj, dict):
115 # First, validate that the dict won't lose data in conversion due to
117 # First, validate that the dict won't lose data in conversion due to
116 # key collisions after stringification. This can happen with keys like
118 # key collisions after stringification. This can happen with keys like
117 # True and 'true' or 1 and '1', which collide in JSON.
119 # True and 'true' or 1 and '1', which collide in JSON.
118 nkeys = len(obj)
120 nkeys = len(obj)
119 nkeys_collapsed = len(set(map(str, obj)))
121 nkeys_collapsed = len(set(map(str, obj)))
120 if nkeys != nkeys_collapsed:
122 if nkeys != nkeys_collapsed:
121 raise ValueError('dict can not be safely converted to JSON: '
123 raise ValueError('dict can not be safely converted to JSON: '
122 'key collision would lead to dropped values')
124 'key collision would lead to dropped values')
123 # If all OK, proceed by making the new dict that will be json-safe
125 # If all OK, proceed by making the new dict that will be json-safe
124 out = {}
126 out = {}
125 for k,v in obj.iteritems():
127 for k,v in obj.iteritems():
126 out[str(k)] = json_clean(v)
128 out[str(k)] = json_clean(v)
127 return out
129 return out
128
130
129 # If we get here, we don't know how to handle the object, so we just get
131 # If we get here, we don't know how to handle the object, so we just get
130 # its repr and return that. This will catch lambdas, open sockets, class
132 # its repr and return that. This will catch lambdas, open sockets, class
131 # objects, and any other complicated contraption that json can't encode
133 # objects, and any other complicated contraption that json can't encode
132 return repr(obj)
134 return repr(obj)
General Comments 0
You need to be logged in to leave comments. Login now