diff --git a/IPython/config/application.py b/IPython/config/application.py index f42567e..d431fdf 100644 --- a/IPython/config/application.py +++ b/IPython/config/application.py @@ -31,7 +31,7 @@ from IPython.external.decorator import decorator from IPython.config.configurable import SingletonConfigurable from IPython.config.loader import ( - KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, + KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, JSONFileConfigLoader ) from IPython.utils.traitlets import ( @@ -492,34 +492,52 @@ class Application(SingletonConfigurable): # flatten flags&aliases, so cl-args get appropriate priority: flags,aliases = self.flatten_flags() - loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, - flags=flags) + flags=flags, log=self.log) config = loader.load_config() self.update_config(config) # store unparsed args in extra_args self.extra_args = loader.extra_args + @classmethod + def _load_config_file(cls, basefilename, path=None, log=None): + """Load config files (json/py) by filename and path.""" + + pyloader = PyFileConfigLoader(basefilename+'.py', path=path, log=log) + jsonloader = JSONFileConfigLoader(basefilename+'.json', path=path, log=log) + config_found = False + config = None + for loader in [pyloader, jsonloader]: + try: + config = loader.load_config() + config_found = True + except ConfigFileNotFound: + pass + except Exception: + # try to get the full filename, but it will be empty in the + # unlikely event that the error raised before filefind finished + filename = loader.full_filename or filename + # problem while running the file + log.error("Exception while loading config file %s", + filename, exc_info=True) + else: + log.debug("Loaded config file: %s", loader.full_filename) + if config : + yield config + + if not config_found : + raise ConfigFileNotFound('Neither .json, not .py file found.') + raise StopIteration + + @catch_config_error def load_config_file(self, filename, path=None): - """Load a .py based config file by filename and path.""" - loader = PyFileConfigLoader(filename, path=path) - try: - config = loader.load_config() - except ConfigFileNotFound: - # problem finding the file, raise - raise - except Exception: - # try to get the full filename, but it will be empty in the - # unlikely event that the error raised before filefind finished - filename = loader.full_filename or filename - # problem while running the file - self.log.error("Exception while loading config file %s", - filename, exc_info=True) - else: - self.log.debug("Loaded config file: %s", loader.full_filename) + """Load config files (json/py) by filename and path.""" + filename, ext = os.path.splitext(filename) + for config in self._load_config_file(filename, path=path , log=self.log): self.update_config(config) + def generate_config_file(self): """generate default config file from Configurables""" lines = ["# Configuration file for %s."%self.name] diff --git a/IPython/config/loader.py b/IPython/config/loader.py index f6e95d8..3d1a60a 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -28,9 +28,10 @@ import copy import os import re import sys +import json from IPython.utils.path import filefind, get_ipython_dir -from IPython.utils import py3compat, warn +from IPython.utils import py3compat from IPython.utils.encoding import DEFAULT_ENCODING from IPython.utils.py3compat import unicode_type, iteritems from IPython.utils.traitlets import HasTraits, List, Any, TraitError @@ -303,9 +304,17 @@ class ConfigLoader(object): handled elsewhere. """ - def __init__(self): + def _log_default(self): + from IPython.config.application import Application + return Application.instance().log + + def __init__(self, log=None): """A base class for config loaders. + log : instance of :class:`logging.Logger` to use. + By default loger of :meth:`IPython.config.application.Application.instance()` + will be used + Examples -------- @@ -315,6 +324,11 @@ class ConfigLoader(object): {} """ self.clear() + if log is None : + self.log = self._log_default() + self.log.debug('Using default logger') + else : + self.log = log def clear(self): self.config = Config() @@ -336,17 +350,8 @@ class FileConfigLoader(ConfigLoader): As we add more file based config loaders, the common logic should go here. """ - pass - - -class PyFileConfigLoader(FileConfigLoader): - """A config loader for pure python files. - - This calls execfile on a plain python file and looks for attributes - that are all caps. These attribute are added to the config Struct. - """ - def __init__(self, filename, path=None): + def __init__(self, filename, path=None, **kw): """Build a config loader for a filename and path. Parameters @@ -357,11 +362,53 @@ class PyFileConfigLoader(FileConfigLoader): The path to search for the config file on, or a sequence of paths to try in order. """ - super(PyFileConfigLoader, self).__init__() + super(FileConfigLoader, self).__init__(**kw) self.filename = filename self.path = path self.full_filename = '' - self.data = None + + def _find_file(self): + """Try to find the file by searching the paths.""" + self.full_filename = filefind(self.filename, self.path) + +class JSONFileConfigLoader(FileConfigLoader): + """A Json file loader for config""" + + def load_config(self): + """Load the config from a file and return it as a Struct.""" + self.clear() + try: + self._find_file() + except IOError as e: + raise ConfigFileNotFound(str(e)) + dct = self._read_file_as_dict() + self.config = self._convert_to_config(dct) + return self.config + + def _read_file_as_dict(self): + with open(self.full_filename) as f : + return json.load(f) + + def _convert_to_config(self, dictionary): + if 'version' in dictionary: + version = dictionary.pop('version') + else : + version = 1 + self.log.warn("Unrecognized JSON config file version, assuming version : {}".format(version)) + + if version == 1: + return Config(dictionary) + else : + raise ValueError('Unknown version of JSON config file : version number {version}'.format(version=version)) + + +class PyFileConfigLoader(FileConfigLoader): + """A config loader for pure python files. + + This is responsible for locating a Python config file by filename and + profile name, then executing it in a namespace where it could have access + to subconfigs. + """ def load_config(self): """Load the config from a file and return it as a Struct.""" @@ -371,12 +418,8 @@ class PyFileConfigLoader(FileConfigLoader): except IOError as e: raise ConfigFileNotFound(str(e)) self._read_file_as_dict() - self._convert_to_config() return self.config - def _find_file(self): - """Try to find the file by searching the paths.""" - self.full_filename = filefind(self.filename, self.path) def _read_file_as_dict(self): """Load the config file into self.config, with recursive loading.""" @@ -429,10 +472,6 @@ class PyFileConfigLoader(FileConfigLoader): conf_filename = self.full_filename.encode(fs_encoding) py3compat.execfile(conf_filename, namespace) - def _convert_to_config(self): - if self.data is None: - ConfigLoaderError('self.data does not exist') - class CommandLineConfigLoader(ConfigLoader): """A config loader for command line arguments. @@ -497,7 +536,7 @@ class KeyValueConfigLoader(CommandLineConfigLoader): ipython --profile="foo" --InteractiveShell.autocall=False """ - def __init__(self, argv=None, aliases=None, flags=None): + def __init__(self, argv=None, aliases=None, flags=None, **kw): """Create a key value pair config loader. Parameters @@ -529,7 +568,7 @@ class KeyValueConfigLoader(CommandLineConfigLoader): >>> sorted(d.items()) [('A', {'name': 'brian'}), ('B', {'number': 0})] """ - self.clear() + super(KeyValueConfigLoader, self).__init__(**kw) if argv is None: argv = sys.argv[1:] self.argv = argv @@ -606,7 +645,7 @@ class KeyValueConfigLoader(CommandLineConfigLoader): lhs = aliases[lhs] if '.' not in lhs: # probably a mistyped alias, but not technically illegal - warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs) + self.log.warn("Unrecognized alias: '%s', it will probably have no effect. %s,-- %s"%(lhs,raw, aliases)) try: self._exec_config_str(lhs, rhs) except Exception: @@ -633,7 +672,7 @@ class KeyValueConfigLoader(CommandLineConfigLoader): class ArgParseConfigLoader(CommandLineConfigLoader): """A loader that uses the argparse module to load from the command line.""" - def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw): + def __init__(self, argv=None, aliases=None, flags=None, log=None, *parser_args, **parser_kw): """Create a config loader for use with argparse. Parameters @@ -656,7 +695,7 @@ class ArgParseConfigLoader(CommandLineConfigLoader): config : Config The resulting Config object. """ - super(CommandLineConfigLoader, self).__init__() + super(CommandLineConfigLoader, self).__init__(log=log) self.clear() if argv is None: argv = sys.argv[1:] @@ -772,7 +811,7 @@ class KVArgParseConfigLoader(ArgParseConfigLoader): self._load_flag(subc) if self.extra_args: - sub_parser = KeyValueConfigLoader() + sub_parser = KeyValueConfigLoader(log=self.log) sub_parser.load_config(self.extra_args) self.config.merge(sub_parser.config) self.extra_args = sub_parser.extra_args diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py index ab455a0..b9e8076 100644 --- a/IPython/config/tests/test_loader.py +++ b/IPython/config/tests/test_loader.py @@ -22,17 +22,21 @@ Authors: import os import pickle import sys +import json + from tempfile import mkstemp from unittest import TestCase from nose import SkipTest +import nose.tools as nt + -from IPython.testing.tools import mute_warn from IPython.config.loader import ( Config, LazyConfigValue, PyFileConfigLoader, + JSONFileConfigLoader, KeyValueConfigLoader, ArgParseConfigLoader, KVArgParseConfigLoader, @@ -53,21 +57,77 @@ c.Foo.Bam.value=list(range(10)) # list() is just so it's the same on Python 3 c.D.C.value='hi there' """ -class TestPyFileCL(TestCase): +json1file = """ +{ + "version": 1, + "a": 10, + "b": 20, + "Foo": { + "Bam": { + "value": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] + }, + "Bar": { + "value": 10 + } + }, + "D": { + "C": { + "value": "hi there" + } + } +} +""" - def test_basic(self): +# should not load +json2file = """ +{ + "version": 2 +} +""" + +import logging +log = logging.getLogger('devnull') +log.setLevel(0) + +class TestFileCL(TestCase): + + def _check_conf(self, config): + self.assertEqual(config.a, 10) + self.assertEqual(config.b, 20) + self.assertEqual(config.Foo.Bar.value, 10) + self.assertEqual(config.Foo.Bam.value, list(range(10))) + self.assertEqual(config.D.C.value, 'hi there') + + def test_python(self): fd, fname = mkstemp('.py') f = os.fdopen(fd, 'w') f.write(pyfile) f.close() # Unlink the file - cl = PyFileConfigLoader(fname) + cl = PyFileConfigLoader(fname, log=log) config = cl.load_config() - self.assertEqual(config.a, 10) - self.assertEqual(config.b, 20) - self.assertEqual(config.Foo.Bar.value, 10) - self.assertEqual(config.Foo.Bam.value, list(range(10))) - self.assertEqual(config.D.C.value, 'hi there') + self._check_conf(config) + + def test_json(self): + fd, fname = mkstemp('.json') + f = os.fdopen(fd, 'w') + f.write(json1file) + f.close() + # Unlink the file + cl = JSONFileConfigLoader(fname, log=log) + config = cl.load_config() + self._check_conf(config) + + def test_v2raise(self): + fd, fname = mkstemp('.json') + f = os.fdopen(fd, 'w') + f.write(json2file) + f.close() + # Unlink the file + cl = JSONFileConfigLoader(fname, log=log) + with nt.assert_raises(ValueError): + cl.load_config() + class MyLoader1(ArgParseConfigLoader): def _add_arguments(self, aliases=None, flags=None): @@ -121,10 +181,9 @@ class TestKeyValueCL(TestCase): klass = KeyValueConfigLoader def test_basic(self): - cl = self.klass() + cl = self.klass(log=log) argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]] - with mute_warn(): - config = cl.load_config(argv) + config = cl.load_config(argv) self.assertEqual(config.a, 10) self.assertEqual(config.b, 20) self.assertEqual(config.Foo.Bar.value, 10) @@ -132,31 +191,27 @@ class TestKeyValueCL(TestCase): self.assertEqual(config.D.C.value, 'hi there') def test_expanduser(self): - cl = self.klass() + cl = self.klass(log=log) argv = ['--a=~/1/2/3', '--b=~', '--c=~/', '--d="~/"'] - with mute_warn(): - config = cl.load_config(argv) + config = cl.load_config(argv) self.assertEqual(config.a, os.path.expanduser('~/1/2/3')) self.assertEqual(config.b, os.path.expanduser('~')) self.assertEqual(config.c, os.path.expanduser('~/')) self.assertEqual(config.d, '~/') def test_extra_args(self): - cl = self.klass() - with mute_warn(): - config = cl.load_config(['--a=5', 'b', '--c=10', 'd']) + cl = self.klass(log=log) + config = cl.load_config(['--a=5', 'b', '--c=10', 'd']) self.assertEqual(cl.extra_args, ['b', 'd']) self.assertEqual(config.a, 5) self.assertEqual(config.c, 10) - with mute_warn(): - config = cl.load_config(['--', '--a=5', '--c=10']) + config = cl.load_config(['--', '--a=5', '--c=10']) self.assertEqual(cl.extra_args, ['--a=5', '--c=10']) def test_unicode_args(self): - cl = self.klass() + cl = self.klass(log=log) argv = [u'--a=épsîlön'] - with mute_warn(): - config = cl.load_config(argv) + config = cl.load_config(argv) self.assertEqual(config.a, u'épsîlön') def test_unicode_bytes_args(self): @@ -166,16 +221,14 @@ class TestKeyValueCL(TestCase): except (TypeError, UnicodeEncodeError): raise SkipTest("sys.stdin.encoding can't handle 'é'") - cl = self.klass() - with mute_warn(): - config = cl.load_config([barg]) + cl = self.klass(log=log) + config = cl.load_config([barg]) self.assertEqual(config.a, u'é') def test_unicode_alias(self): - cl = self.klass() + cl = self.klass(log=log) argv = [u'--a=épsîlön'] - with mute_warn(): - config = cl.load_config(argv, aliases=dict(a='A.a')) + config = cl.load_config(argv, aliases=dict(a='A.a')) self.assertEqual(config.A.a, u'épsîlön') @@ -183,18 +236,16 @@ class TestArgParseKVCL(TestKeyValueCL): klass = KVArgParseConfigLoader def test_expanduser2(self): - cl = self.klass() + cl = self.klass(log=log) argv = ['-a', '~/1/2/3', '--b', "'~/1/2/3'"] - with mute_warn(): - config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b')) + config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b')) self.assertEqual(config.A.a, os.path.expanduser('~/1/2/3')) self.assertEqual(config.A.b, '~/1/2/3') def test_eval(self): - cl = self.klass() + cl = self.klass(log=log) argv = ['-c', 'a=5'] - with mute_warn(): - config = cl.load_config(argv, aliases=dict(c='A.c')) + config = cl.load_config(argv, aliases=dict(c='A.c')) self.assertEqual(config.A.c, u"a=5") diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index f9dd677..6026a2f 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -33,7 +33,7 @@ import sys from IPython.config.loader import ( Config, PyFileConfigLoader, ConfigFileNotFound ) -from IPython.config.application import boolean_flag, catch_config_error +from IPython.config.application import boolean_flag, catch_config_error, Application from IPython.core import release from IPython.core import usage from IPython.core.completer import IPCompleter @@ -364,7 +364,6 @@ class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): else: self.log.debug("IPython not interactive...") - def load_default_config(ipython_dir=None): """Load the default config file from the default ipython_dir. @@ -372,15 +371,14 @@ def load_default_config(ipython_dir=None): """ if ipython_dir is None: ipython_dir = get_ipython_dir() + profile_dir = os.path.join(ipython_dir, 'profile_default') - cl = PyFileConfigLoader("ipython_config.py", profile_dir) - try: - config = cl.load_config() - except ConfigFileNotFound: - # no config found - config = Config() - return config + config = Config() + for cf in Application._load_config_file(filename[:-3], path=profile_dir, log=None): + config.update(cf) + + return config launch_new_instance = TerminalIPythonApp.launch_instance diff --git a/docs/source/development/config.rst b/docs/source/development/config.rst index d92e377..674ad5b 100644 --- a/docs/source/development/config.rst +++ b/docs/source/development/config.rst @@ -15,10 +15,10 @@ Each of these abstractions is represented by a Python class. Configuration object: :class:`~IPython.config.loader.Config` A configuration object is a simple dictionary-like class that holds configuration attributes and sub-configuration objects. These classes - support dotted attribute style access (``Foo.bar``) in addition to the - regular dictionary style access (``Foo['bar']``). Configuration objects - are smart. They know how to merge themselves with other configuration - objects and they automatically create sub-configuration objects. + support dotted attribute style access (``cfg.Foo.bar``) in addition to the + regular dictionary style access (``cfg['Foo']['bar']``). + The Config object is a wrapper around a simple dictionary with some convenience methods, + such as merging and automatic section creation. Application: :class:`~IPython.config.application.Application` An application is a process that does a specific job. The most obvious @@ -85,12 +85,24 @@ Now, we show what our configuration objects and files look like. Configuration objects and files =============================== -A configuration file is simply a pure Python file that sets the attributes -of a global, pre-created configuration object. This configuration object is a -:class:`~IPython.config.loader.Config` instance. While in a configuration -file, to get a reference to this object, simply call the :func:`get_config` -function. We inject this function into the global namespace that the -configuration file is executed in. +A configuration object is little more than a wrapper around a dictionary. +A configuration *file* is simply a mechanism for producing that object. +The main IPython configuration file is a plain Python script, +which can perform extensive logic to populate the config object. +IPython 2.0 introduces a JSON configuration file, +which is just a direct JSON serialization of the config dictionary. +The JSON format is easily processed by external software. + +When both Python and JSON configuration file are present, both will be loaded, +with JSON configuration having higher priority. + +Python configuration Files +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A Python configuration file is a pure Python file that populates a configuration object. +This configuration object is a :class:`~IPython.config.loader.Config` instance. +While in a configuration file, to get a reference to this object, simply call the :func:`get_config` +function, which is available in the global namespace of the script. Here is an example of a super simple configuration file that does nothing:: @@ -99,10 +111,11 @@ Here is an example of a super simple configuration file that does nothing:: Once you get a reference to the configuration object, you simply set attributes on it. All you have to know is: -* The name of each attribute. +* The name of the class to configure. +* The name of the attribute. * The type of each attribute. -The answers to these two questions are provided by the various +The answers to these questions are provided by the various :class:`~IPython.config.configurable.Configurable` subclasses that an application uses. Let's look at how this would work for a simple configurable subclass:: @@ -118,7 +131,7 @@ subclass:: # The rest of the class implementation would go here.. In this example, we see that :class:`MyClass` has three attributes, two -of whom (``name``, ``ranking``) can be configured. All of the attributes +of (``name``, ``ranking``) can be configured. All of the attributes are given types and default values. If a :class:`MyClass` is instantiated, but not configured, these default values will be used. But let's see how to configure this class in a configuration file:: @@ -173,9 +186,38 @@ attribute of ``c`` is not the actual class, but instead is another instance is dynamically created for that attribute. This allows deeply hierarchical information created easily (``c.Foo.Bar.value``) on the fly. +JSON configuration Files +~~~~~~~~~~~~~~~~~~~~~~~~ + +A JSON configuration file is simply a file that contain a +:class:`~IPython.config.loader.Config` dictionary serialized to JSON. +A JSON configuration file has the same base name as a Python configuration file, +just with a .json extension. + +Configuration described in previous section could be written as follow in a +JSON configuration file: + +.. sourcecode:: json + + { + "version": "1.0", + "MyClass": { + "name": "coolname", + "ranking": 10 + } + } + +JSON configuration files can be more easily generated or processed by programs +or other languages. + + Configuration files inheritance =============================== +.. note:: + + This section only apply to Python configuration files. + Let's say you want to have different configuration files for various purposes. Our configuration system makes it easy for one configuration file to inherit the information in another configuration file. The :func:`load_subconfig` diff --git a/docs/source/whatsnew/pr/jsonconfig.rst b/docs/source/whatsnew/pr/jsonconfig.rst new file mode 100644 index 0000000..afe5a13 --- /dev/null +++ b/docs/source/whatsnew/pr/jsonconfig.rst @@ -0,0 +1,3 @@ +* IPython config objects can be loaded from and serialized to JSON. + JSON config file have the same base name as their ``.py`` counterpart, + and will be loaded with higher priority if found.