From 5c6d229ce5f249ef1f122da2ae813b70e14a3c40 2010-09-04 08:27:22 From: Fernando Perez Date: 2010-09-04 08:27:22 Subject: [PATCH] Created JSON-safety utilities. These will be useful to make it easy to create objects that are safe for encoding as JSON. Full test suite with 100% coverage included. --- diff --git a/IPython/utils/jsonutil.py b/IPython/utils/jsonutil.py new file mode 100644 index 0000000..f7b1f76 --- /dev/null +++ b/IPython/utils/jsonutil.py @@ -0,0 +1,90 @@ +"""Utilities to manipulate JSON objects. +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING.txt, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +# stdlib +import types + +#----------------------------------------------------------------------------- +# Classes and functions +#----------------------------------------------------------------------------- + +def json_clean(obj): + """Clean an object to ensure it's safe to encode in JSON. + + Atomic, immutable objects are returned unmodified. Sets and tuples are + converted to lists, lists are copied and dicts are also copied. + + Note: dicts whose keys could cause collisions upon encoding (such as a dict + with both the number 1 and the string '1' as keys) will cause a ValueError + to be raised. + + Parameters + ---------- + obj : any python object + + Returns + ------- + out : object + + A version of the input which will not cause an encoding error when + encoded as JSON. Note that this function does not *encode* its inputs, + it simply sanitizes it so that there will be no encoding errors later. + + Examples + -------- + >>> json_clean(4) + 4 + >>> json_clean(range(10)) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> json_clean(dict(x=1, y=2)) + {'y': 2, 'x': 1} + >>> json_clean(dict(x=1, y=2, z=[1,2,3])) + {'y': 2, 'x': 1, 'z': [1, 2, 3]} + >>> json_clean(True) + True + """ + # types that are 'atomic' and ok in json as-is. bool doesn't need to be + # listed explicitly because bools pass as int instances + atomic_ok = (basestring, int, float, types.NoneType) + + # containers that we need to convert into lists + container_to_list = (tuple, set, types.GeneratorType) + + if isinstance(obj, atomic_ok): + return obj + + if isinstance(obj, container_to_list) or ( + hasattr(obj, '__iter__') and hasattr(obj, 'next')): + obj = list(obj) + + if isinstance(obj, list): + return [json_clean(x) for x in obj] + + if isinstance(obj, dict): + # First, validate that the dict won't lose data in conversion due to + # key collisions after stringification. This can happen with keys like + # True and 'true' or 1 and '1', which collide in JSON. + nkeys = len(obj) + nkeys_collapsed = len(set(map(str, obj))) + if nkeys != nkeys_collapsed: + raise ValueError('dict can not be safely converted to JSON: ' + 'key collision would lead to dropped values') + # If all OK, proceed by making the new dict that will be json-safe + out = {} + for k,v in obj.iteritems(): + out[str(k)] = json_clean(v) + return out + + # If we get here, we don't know how to handle the object, so we just get + # its repr and return that. This will catch lambdas, open sockets, class + # objects, and any other complicated contraption that json can't encode + return repr(obj) diff --git a/IPython/utils/tests/test_jsonutil.py b/IPython/utils/tests/test_jsonutil.py new file mode 100644 index 0000000..d81eafc --- /dev/null +++ b/IPython/utils/tests/test_jsonutil.py @@ -0,0 +1,71 @@ +"""Test suite for our JSON utilities. +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING.txt, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +# stdlib +import json + +# third party +import nose.tools as nt + +# our own +from ..jsonutil import json_clean + +#----------------------------------------------------------------------------- +# Test functions +#----------------------------------------------------------------------------- + +def test(): + # list of input/expected output. Use None for the expected output if it + # can be the same as the input. + pairs = [(1, None), # start with scalars + (1.0, None), + ('a', None), + (True, None), + (False, None), + (None, None), + # complex numbers for now just go to strings, as otherwise they + # are unserializable + (1j, '1j'), + # Containers + ([1, 2], None), + ((1, 2), [1, 2]), + (set([1, 2]), [1, 2]), + (dict(x=1), None), + ({'x': 1, 'y':[1,2,3], '1':'int'}, None), + # More exotic objects + ((x for x in range(3)), [0, 1, 2]), + (iter([1, 2]), [1, 2]), + ] + + for val, jval in pairs: + if jval is None: + jval = val + out = json_clean(val) + # validate our cleanup + nt.assert_equal(out, jval) + # and ensure that what we return, indeed encodes cleanly + json.loads(json.dumps(out)) + + +def test_lambda(): + jc = json_clean(lambda : 1) + nt.assert_true(jc.startswith(' at ')) + json.dumps(jc) + + +def test_exception(): + bad_dicts = [{1:'number', '1':'string'}, + {True:'bool', 'True':'string'}, + ] + for d in bad_dicts: + nt.assert_raises(ValueError, json_clean, d) +