allow to load config from json file

json takes highest precedence and is loaded last

docs/source/whatsnew/pr/jsonconfig.rst

@@ -0,0 +1,3
+* IPython config objects can be loaded from and serialized to JSON.
+  JSON config file have the same base name as their ``.py`` counterpart,
+  and will be loaded with higher priority if found.

IPython/config/application.py

@@ -31,7 +31,7 from IPython.external.decorator import decorator
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.config.loader import (
-    KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
+    KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, JSONFileConfigLoader
 )
 
 from IPython.utils.traitlets import (
@@ -492,34 +492,52 class Application(SingletonConfigurable):
         
         # flatten flags&aliases, so cl-args get appropriate priority:
         flags,aliases = self.flatten_flags()
-
         loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
-                                        flags=flags)
+                                        flags=flags, log=self.log)
         config = loader.load_config()
         self.update_config(config)
         # store unparsed args in extra_args
         self.extra_args = loader.extra_args
 
+    @classmethod
+    def _load_config_file(cls, basefilename, path=None, log=None):
+        """Load config files (json/py) by filename and path."""
+
+        pyloader = PyFileConfigLoader(basefilename+'.py', path=path, log=log)
+        jsonloader = JSONFileConfigLoader(basefilename+'.json', path=path, log=log)
+        config_found = False
+        config = None
+        for loader in [pyloader, jsonloader]:
+            try:
+                config = loader.load_config()
+                config_found = True
+            except ConfigFileNotFound:
+                pass
+            except Exception:
+                # try to get the full filename, but it will be empty in the
+                # unlikely event that the error raised before filefind finished
+                filename = loader.full_filename or filename
+                # problem while running the file
+                log.error("Exception while loading config file %s",
+                                filename, exc_info=True)
+            else:
+                log.debug("Loaded config file: %s", loader.full_filename)
+            if config :
+                 yield config
+
+        if not config_found :
+            raise ConfigFileNotFound('Neither .json, not .py file found.')
+        raise StopIteration
+
+
     @catch_config_error
     def load_config_file(self, filename, path=None):
-        """Load a .py based config file by filename and path."""
-        loader = PyFileConfigLoader(filename, path=path)
-        try:
-            config = loader.load_config()
-        except ConfigFileNotFound:
-            # problem finding the file, raise
-            raise
-        except Exception:
-            # try to get the full filename, but it will be empty in the
-            # unlikely event that the error raised before filefind finished
-            filename = loader.full_filename or filename
-            # problem while running the file
-            self.log.error("Exception while loading config file %s",
-                            filename, exc_info=True)
-        else:
-            self.log.debug("Loaded config file: %s", loader.full_filename)
+        """Load config files (json/py) by filename and path."""
+        filename, ext = os.path.splitext(filename)
+        for config in self._load_config_file(filename, path=path , log=self.log):
             self.update_config(config)
 
+
     def generate_config_file(self):
         """generate default config file from Configurables"""
         lines = ["# Configuration file for %s."%self.name]

IPython/config/loader.py

@@ -28,9 +28,10 import copy
 import os
 import re
 import sys
+import json
 
 from IPython.utils.path import filefind, get_ipython_dir
-from IPython.utils import py3compat, warn
+from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 from IPython.utils.py3compat import unicode_type, iteritems
 from IPython.utils.traitlets import HasTraits, List, Any, TraitError
@@ -303,9 +304,17 class ConfigLoader(object):
     handled elsewhere.
     """
 
-    def __init__(self):
+    def _log_default(self):
+        from IPython.config.application import Application
+        return Application.instance().log
+
+    def __init__(self, log=None):
         """A base class for config loaders.
 
+        log : instance of :class:`logging.Logger` to use.
+              By default loger of :meth:`IPython.config.application.Application.instance()`
+              will be used
+
         Examples
         --------
 
@@ -315,6 +324,11 class ConfigLoader(object):
         {}
         """
         self.clear()
+        if log is None :
+            self.log = self._log_default()
+            self.log.debug('Using default logger')
+        else :
+            self.log = log
 
     def clear(self):
         self.config = Config()
@@ -336,17 +350,8 class FileConfigLoader(ConfigLoader):
     As we add more file based config loaders, the common logic should go
     here.
     """
-    pass
-
-
-class PyFileConfigLoader(FileConfigLoader):
-    """A config loader for pure python files.
-
-    This calls execfile on a plain python file and looks for attributes
-    that are all caps.  These attribute are added to the config Struct.
-    """
 
-    def __init__(self, filename, path=None):
+    def __init__(self, filename, path=None, **kw):
         """Build a config loader for a filename and path.
 
         Parameters
@@ -357,11 +362,53 class PyFileConfigLoader(FileConfigLoader):
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
-        super(PyFileConfigLoader, self).__init__()
+        super(FileConfigLoader, self).__init__(**kw)
         self.filename = filename
         self.path = path
         self.full_filename = ''
-        self.data = None
+
+    def _find_file(self):
+        """Try to find the file by searching the paths."""
+        self.full_filename = filefind(self.filename, self.path)
+
+class JSONFileConfigLoader(FileConfigLoader):
+    """A Json file loader for config"""
+
+    def load_config(self):
+        """Load the config from a file and return it as a Struct."""
+        self.clear()
+        try:
+            self._find_file()
+        except IOError as e:
+            raise ConfigFileNotFound(str(e))
+        dct = self._read_file_as_dict()
+        self.config = self._convert_to_config(dct)
+        return self.config
+
+    def _read_file_as_dict(self):
+        with open(self.full_filename) as f :
+            return json.load(f)
+
+    def _convert_to_config(self, dictionary):
+        if 'version' in dictionary:
+            version = dictionary.pop('version')
+        else :
+            version = 1
+            self.log.warn("Unrecognized JSON config file version, assuming version : {}".format(version))
+
+        if version == 1:
+            return Config(dictionary)
+        else :
+            raise ValueError('Unknown version of JSON config file : version number {version}'.format(version=version))
+
+
+class PyFileConfigLoader(FileConfigLoader):
+    """A config loader for pure python files.
+
+    This is responsible for locating a Python config file by filename and
+    profile name, then executing it in a namespace where it could have access
+    to subconfigs.
+    """
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
@@ -371,12 +418,8 class PyFileConfigLoader(FileConfigLoader):
         except IOError as e:
             raise ConfigFileNotFound(str(e))
         self._read_file_as_dict()
-        self._convert_to_config()
         return self.config
 
-    def _find_file(self):
-        """Try to find the file by searching the paths."""
-        self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
@@ -429,10 +472,6 class PyFileConfigLoader(FileConfigLoader):
         conf_filename = self.full_filename.encode(fs_encoding)
         py3compat.execfile(conf_filename, namespace)
 
-    def _convert_to_config(self):
-        if self.data is None:
-            ConfigLoaderError('self.data does not exist')
-
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
@@ -497,7 +536,7 class KeyValueConfigLoader(CommandLineConfigLoader):
         ipython --profile="foo" --InteractiveShell.autocall=False
     """
 
-    def __init__(self, argv=None, aliases=None, flags=None):
+    def __init__(self, argv=None, aliases=None, flags=None, **kw):
         """Create a key value pair config loader.
 
         Parameters
@@ -529,7 +568,7 class KeyValueConfigLoader(CommandLineConfigLoader):
             >>> sorted(d.items())
             [('A', {'name': 'brian'}), ('B', {'number': 0})]
         """
-        self.clear()
+        super(KeyValueConfigLoader, self).__init__(**kw)
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
@@ -606,7 +645,7 class KeyValueConfigLoader(CommandLineConfigLoader):
                     lhs = aliases[lhs]
                 if '.' not in lhs:
                     # probably a mistyped alias, but not technically illegal
-                    warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
+                    self.log.warn("Unrecognized alias: '%s', it will probably have no effect. %s,-- %s"%(lhs,raw, aliases))
                 try:
                     self._exec_config_str(lhs, rhs)
                 except Exception:
@@ -633,7 +672,7 class KeyValueConfigLoader(CommandLineConfigLoader):
 class ArgParseConfigLoader(CommandLineConfigLoader):
     """A loader that uses the argparse module to load from the command line."""
 
-    def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
+    def __init__(self, argv=None, aliases=None, flags=None, log=None,  *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
@@ -656,7 +695,7 class ArgParseConfigLoader(CommandLineConfigLoader):
         config : Config
             The resulting Config object.
         """
-        super(CommandLineConfigLoader, self).__init__()
+        super(CommandLineConfigLoader, self).__init__(log=log)
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
@@ -772,7 +811,7 class KVArgParseConfigLoader(ArgParseConfigLoader):
             self._load_flag(subc)
 
         if self.extra_args:
-            sub_parser = KeyValueConfigLoader()
+            sub_parser = KeyValueConfigLoader(log=self.log)
             sub_parser.load_config(self.extra_args)
             self.config.merge(sub_parser.config)
             self.extra_args = sub_parser.extra_args

IPython/config/tests/test_loader.py

@@ -22,17 +22,21 Authors:
 import os
 import pickle
 import sys
+import json
+
 from tempfile import mkstemp
 from unittest import TestCase
 
 from nose import SkipTest
+import nose.tools as nt
+
 
-from IPython.testing.tools import mute_warn
 
 from IPython.config.loader import (
     Config,
     LazyConfigValue,
     PyFileConfigLoader,
+    JSONFileConfigLoader,
     KeyValueConfigLoader,
     ArgParseConfigLoader,
     KVArgParseConfigLoader,
@@ -53,21 +57,77 c.Foo.Bam.value=list(range(10)) # list() is just so it's the same on Python 3
 c.D.C.value='hi there'
 """
 
-class TestPyFileCL(TestCase):
+json1file = """
+{
+  "version": 1,
+  "a": 10,
+  "b": 20,
+  "Foo": {
+    "Bam": {
+      "value": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]
+    },
+    "Bar": {
+      "value": 10
+    }
+  },
+  "D": {
+    "C": {
+      "value": "hi there"
+    }
+  }
+}
+"""
 
-    def test_basic(self):
+# should not load
+json2file = """
+{
+  "version": 2
+}
+"""
+
+import logging
+log = logging.getLogger('devnull')
+log.setLevel(0)
+
+class TestFileCL(TestCase):
+
+    def _check_conf(self, config):
+        self.assertEqual(config.a, 10)
+        self.assertEqual(config.b, 20)
+        self.assertEqual(config.Foo.Bar.value, 10)
+        self.assertEqual(config.Foo.Bam.value, list(range(10)))
+        self.assertEqual(config.D.C.value, 'hi there')
+
+    def test_python(self):
         fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
         # Unlink the file
-        cl = PyFileConfigLoader(fname)
+        cl = PyFileConfigLoader(fname, log=log)
         config = cl.load_config()
-        self.assertEqual(config.a, 10)
-        self.assertEqual(config.b, 20)
-        self.assertEqual(config.Foo.Bar.value, 10)
-        self.assertEqual(config.Foo.Bam.value, list(range(10)))
-        self.assertEqual(config.D.C.value, 'hi there')
+        self._check_conf(config)
+
+    def test_json(self):
+        fd, fname = mkstemp('.json')
+        f = os.fdopen(fd, 'w')
+        f.write(json1file)
+        f.close()
+        # Unlink the file
+        cl = JSONFileConfigLoader(fname, log=log)
+        config = cl.load_config()
+        self._check_conf(config)
+
+    def test_v2raise(self):
+        fd, fname = mkstemp('.json')
+        f = os.fdopen(fd, 'w')
+        f.write(json2file)
+        f.close()
+        # Unlink the file
+        cl = JSONFileConfigLoader(fname, log=log)
+        with nt.assert_raises(ValueError):
+            cl.load_config()
+
 
 class MyLoader1(ArgParseConfigLoader):
     def _add_arguments(self, aliases=None, flags=None):
@@ -121,10 +181,9 class TestKeyValueCL(TestCase):
     klass = KeyValueConfigLoader
 
     def test_basic(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
-        with mute_warn():
-            config = cl.load_config(argv)
+        config = cl.load_config(argv)
         self.assertEqual(config.a, 10)
         self.assertEqual(config.b, 20)
         self.assertEqual(config.Foo.Bar.value, 10)
@@ -132,31 +191,27 class TestKeyValueCL(TestCase):
         self.assertEqual(config.D.C.value, 'hi there')
     
     def test_expanduser(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = ['--a=~/1/2/3', '--b=~', '--c=~/', '--d="~/"']
-        with mute_warn():
-            config = cl.load_config(argv)
+        config = cl.load_config(argv)
         self.assertEqual(config.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.b, os.path.expanduser('~'))
         self.assertEqual(config.c, os.path.expanduser('~/'))
         self.assertEqual(config.d, '~/')
     
     def test_extra_args(self):
-        cl = self.klass()
-        with mute_warn():
-            config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
+        cl = self.klass(log=log)
+        config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
         self.assertEqual(cl.extra_args, ['b', 'd'])
         self.assertEqual(config.a, 5)
         self.assertEqual(config.c, 10)
-        with mute_warn():
-            config = cl.load_config(['--', '--a=5', '--c=10'])
+        config = cl.load_config(['--', '--a=5', '--c=10'])
         self.assertEqual(cl.extra_args, ['--a=5', '--c=10'])
     
     def test_unicode_args(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = [u'--a=épsîlön']
-        with mute_warn():
-            config = cl.load_config(argv)
+        config = cl.load_config(argv)
         self.assertEqual(config.a, u'épsîlön')
     
     def test_unicode_bytes_args(self):
@@ -166,16 +221,14 class TestKeyValueCL(TestCase):
         except (TypeError, UnicodeEncodeError):
             raise SkipTest("sys.stdin.encoding can't handle 'é'")
         
-        cl = self.klass()
-        with mute_warn():
-            config = cl.load_config([barg])
+        cl = self.klass(log=log)
+        config = cl.load_config([barg])
         self.assertEqual(config.a, u'é')
     
     def test_unicode_alias(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = [u'--a=épsîlön']
-        with mute_warn():
-            config = cl.load_config(argv, aliases=dict(a='A.a'))
+        config = cl.load_config(argv, aliases=dict(a='A.a'))
         self.assertEqual(config.A.a, u'épsîlön')
 
 
@@ -183,18 +236,16 class TestArgParseKVCL(TestKeyValueCL):
     klass = KVArgParseConfigLoader
 
     def test_expanduser2(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = ['-a', '~/1/2/3', '--b', "'~/1/2/3'"]
-        with mute_warn():
-            config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b'))
+        config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b'))
         self.assertEqual(config.A.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.A.b, '~/1/2/3')
     
     def test_eval(self):
-        cl = self.klass()
+        cl = self.klass(log=log)
         argv = ['-c', 'a=5']
-        with mute_warn():
-            config = cl.load_config(argv, aliases=dict(c='A.c'))
+        config = cl.load_config(argv, aliases=dict(c='A.c'))
         self.assertEqual(config.A.c, u"a=5")
     
 

IPython/terminal/ipapp.py

@@ -33,7 +33,7 import sys
 from IPython.config.loader import (
     Config, PyFileConfigLoader, ConfigFileNotFound
 )
-from IPython.config.application import boolean_flag, catch_config_error
+from IPython.config.application import boolean_flag, catch_config_error, Application
 from IPython.core import release
 from IPython.core import usage
 from IPython.core.completer import IPCompleter
@@ -364,7 +364,6 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
         else:
             self.log.debug("IPython not interactive...")
 
-
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
@@ -372,15 +371,14 def load_default_config(ipython_dir=None):
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
+
     profile_dir = os.path.join(ipython_dir, 'profile_default')
-    cl = PyFileConfigLoader("ipython_config.py", profile_dir)
-    try:
-        config = cl.load_config()
-    except ConfigFileNotFound:
-        # no config found
-        config = Config()
-    return config
 
+    config = Config()
+    for cf in Application._load_config_file(filename[:-3], path=profile_dir, log=None):
+        config.update(cf)
+
+    return config
 
 launch_new_instance = TerminalIPythonApp.launch_instance
 

docs/source/development/config.rst

@@ -15,10 +15,10 Each of these abstractions is represented by a Python class.
 Configuration object: :class:`~IPython.config.loader.Config`
     A configuration object is a simple dictionary-like class that holds
     configuration attributes and sub-configuration objects. These classes
-    support dotted attribute style access (``Foo.bar``) in addition to the
-    regular dictionary style access (``Foo['bar']``). Configuration objects
-    are smart. They know how to merge themselves with other configuration
-    objects and they automatically create sub-configuration objects.
+    support dotted attribute style access (``cfg.Foo.bar``) in addition to the
+    regular dictionary style access (``cfg['Foo']['bar']``).
+    The Config object is a wrapper around a simple dictionary with some convenience methods,
+    such as merging and automatic section creation.
 
 Application: :class:`~IPython.config.application.Application`
     An application is a process that does a specific job. The most obvious
@@ -85,12 +85,24 Now, we show what our configuration objects and files look like.
 Configuration objects and files
 ===============================
 
-A configuration file is simply a pure Python file that sets the attributes
-of a global, pre-created configuration object.  This configuration object is a 
-:class:`~IPython.config.loader.Config` instance.  While in a configuration
-file, to get a reference to this object, simply call the :func:`get_config`
-function.  We inject this function into the global namespace that the 
-configuration file is executed in.
+A configuration object is little more than a wrapper around a dictionary.
+A configuration *file* is simply a mechanism for producing that object.
+The main IPython configuration file is a plain Python script,
+which can perform extensive logic to populate the config object.
+IPython 2.0 introduces a JSON configuration file,
+which is just a direct JSON serialization of the config dictionary.
+The JSON format is easily processed by external software.
+
+When both Python and JSON configuration file are present, both will be loaded,
+with JSON configuration having higher priority.
+
+Python configuration Files
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A Python configuration file is a pure Python file that populates a configuration object.
+This configuration object is a :class:`~IPython.config.loader.Config` instance.
+While in a configuration file, to get a reference to this object, simply call the :func:`get_config`
+function, which is available in the global namespace of the script.
 
 Here is an example of a super simple configuration file that does nothing::
 
@@ -99,10 +111,11 Here is an example of a super simple configuration file that does nothing::
 Once you get a reference to the configuration object, you simply set
 attributes on it.  All you have to know is:
 
-* The name of each attribute.
+* The name of the class to configure.
+* The name of the attribute.
 * The type of each attribute.
 
-The answers to these two questions are provided by the various
+The answers to these questions are provided by the various
 :class:`~IPython.config.configurable.Configurable` subclasses that an
 application uses. Let's look at how this would work for a simple configurable
 subclass::
@@ -118,7 +131,7 subclass::
         # The rest of the class implementation would go here..
 
 In this example, we see that :class:`MyClass` has three attributes, two
-of whom (``name``, ``ranking``) can be configured.  All of the attributes
+of  (``name``, ``ranking``) can be configured.  All of the attributes
 are given types and default values.  If a :class:`MyClass` is instantiated,
 but not configured, these default values will be used.  But let's see how
 to configure this class in a configuration file::
@@ -173,9 +186,38 attribute of ``c`` is not the actual class, but instead is another
     instance is dynamically created for that attribute. This allows deeply
     hierarchical information created easily (``c.Foo.Bar.value``) on the fly.
 
+JSON configuration Files
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+A JSON configuration file is simply a file that contain a
+:class:`~IPython.config.loader.Config` dictionary serialized to JSON.
+A JSON configuration file has the same base name as a Python configuration file,
+just with a .json extension.
+
+Configuration described in previous section could be written as follow in a
+JSON configuration file:
+
+.. sourcecode:: json
+
+    {
+      "version": "1.0",
+      "MyClass": {
+        "name": "coolname",
+        "ranking": 10
+      }
+    }
+
+JSON configuration files can be more easily generated or processed by programs
+or other languages.
+
+
 Configuration files inheritance
 ===============================
 
+.. note::
+
+    This section only apply to Python configuration files.
+
 Let's say you want to have different configuration files for various purposes.
 Our configuration system makes it easy for one configuration file to inherit
 the information in another configuration file. The :func:`load_subconfig`