Merge pull request #4276 from minrk/config-extend

Support container methods in config

docs/source/whatsnew/pr/config-extend.rst

@@ -0,0 +1,15 b''
+Extending Configurable Containers
+---------------------------------
+
+Some configurable traits are containers (list, dict, set)
+Config objects now support calling ``extend``, ``update``, ``insert``, etc.
+on traits in config files, which will ultimately result in calling
+those methods on the original object.
+
+The effect being that you can now add to containers without having to copy/paste
+the initial value::
+
+    c = get_config()
+    c.InlineBackend.rc.update({ 'figure.figsize' : (6, 4) })
+
+

IPython/config/configurable.py

@@ -28,7 +28,7 b' Authors:'
 import datetime
 from copy import deepcopy
 
-from loader import Config
+from .loader import Config, LazyConfigValue
 from IPython.utils.traitlets import HasTraits, Instance
 from IPython.utils.text import indent, wrap_paragraphs
 
@@ -137,7 +137,7 b' class Configurable(HasTraits):'
                 if c._has_section(sname):
                     my_config.merge(c[sname])
         return my_config
-
+    
     def _load_config(self, cfg, section_names=None, traits=None):
         """load traits from a Config object"""
         
@@ -149,6 +149,11 b' class Configurable(HasTraits):'
         my_config = self._find_my_config(cfg)
         for name, config_value in my_config.iteritems():
             if name in traits:
+                if isinstance(config_value, LazyConfigValue):
+                    # ConfigValue is a wrapper for using append / update on containers
+                    # without having to copy the 
+                    initial = getattr(self, name)
+                    config_value = config_value.get_value(initial)
                 # We have to do a deepcopy here if we don't deepcopy the entire
                 # config object. If we don't, a mutable config_value will be
                 # shared by all instances, effectively making it a class attribute.

IPython/config/loader.py

@@ -25,6 +25,7 b' Authors'
 
 import __builtin__ as builtin_mod
 import argparse
+import copy
 import os
 import re
 import sys
@@ -32,6 +33,7 b' import sys'
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat, warn
 from IPython.utils.encoding import DEFAULT_ENCODING
+from IPython.utils.traitlets import HasTraits, List, Any, TraitError
 
 #-----------------------------------------------------------------------------
 # Exceptions
@@ -74,6 +76,92 b' class ArgumentParser(argparse.ArgumentParser):'
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
+class LazyConfigValue(HasTraits):
+    """Proxy object for exposing methods on configurable containers
+    
+    Exposes:
+    
+    - append, extend, insert on lists
+    - update on dicts
+    - update, add on sets
+    """
+    
+    _value = None
+    
+    # list methods
+    _extend = List()
+    _prepend = List()
+    
+    def append(self, obj):
+        self._extend.append(obj)
+    
+    def extend(self, other):
+        self._extend.extend(other)
+    
+    def prepend(self, other):
+        """like list.extend, but for the front"""
+        self._prepend[:0] = other
+    
+    _inserts = List()
+    def insert(self, index, other):
+        if not isinstance(index, int):
+            raise TypeError("An integer is required")
+        self._inserts.append((index, other))
+    
+    # dict methods
+    # update is used for both dict and set
+    _update = Any()
+    def update(self, other):
+        if self._update is None:
+            if isinstance(other, dict):
+                self._update = {}
+            else:
+                self._update = set()
+        self._update.update(other)
+    
+    # set methods
+    def add(self, obj):
+        self.update({obj})
+    
+    def get_value(self, initial):
+        """construct the value from the initial one
+        
+        after applying any insert / extend / update changes
+        """
+        if self._value is not None:
+            return self._value
+        value = copy.deepcopy(initial)
+        if isinstance(value, list):
+            for idx, obj in self._inserts:
+                value.insert(idx, obj)
+            value[:0] = self._prepend
+            value.extend(self._extend)
+        
+        elif isinstance(value, dict):
+            if self._update:
+                value.update(self._update)
+        elif isinstance(value, set):
+            if self._update:
+                value.update(self._update)
+        self._value = value
+        return value
+    
+    def to_dict(self):
+        """return JSONable dict form of my data
+        
+        Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
+        """
+        d = {}
+        if self._update:
+            d['update'] = self._update
+        if self._extend:
+            d['extend'] = self._extend
+        if self._prepend:
+            d['prepend'] = self._prepend
+        elif self._inserts:
+            d['inserts'] = self._inserts
+        return d
+
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
@@ -125,6 +213,14 b' class Config(dict):'
             return False
 
     def __contains__(self, key):
+        # allow nested contains of the form `"Section.key" in config`
+        if '.' in key:
+            first, remainder = key.split('.', 1)
+            if first not in self:
+                return False
+            return remainder in self[first]
+        
+        # we always have Sections
         if self._is_section_key(key):
             return True
         else:
@@ -147,7 +243,7 b' class Config(dict):'
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
-
+    
     def __getitem__(self, key):
         # We cannot use directly self._is_section_key, because it triggers
         # infinite recursion on top of PyPy. Instead, we manually fish the
@@ -170,7 +266,14 b' class Config(dict):'
                 dict.__setitem__(self, key, c)
                 return c
         else:
-            return dict.__getitem__(self, key)
+            try:
+                return dict.__getitem__(self, key)
+            except KeyError:
+                # undefined
+                v = LazyConfigValue()
+                dict.__setitem__(self, key, v)
+                return v
+            
 
     def __setitem__(self, key, value):
         if self._is_section_key(key):

IPython/config/profile/cluster/ipython_config.py

@@ -9,11 +9,5 b' lines = """'
 from IPython.parallel import *
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)
 

IPython/config/profile/math/ipython_config.py

@@ -10,12 +10,4 b' import cmath'
 from math import *
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
-
+app.exec_lines.append(lines)

IPython/config/profile/pysh/ipython_config.py

@@ -21,10 +21,4 b' lines = """'
 %rehashx
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)

IPython/config/profile/sympy/ipython_config.py

@@ -13,18 +13,8 b" k, m, n = symbols('k m n', integer=True)"
 f, g, h = symbols('f g h', cls=Function)
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)
 
 # Load the sympy_printing extension to enable nice printing of sympy expr's.
-if hasattr(app, 'extensions'):
-    app.extensions.append('sympyprinting')
-else:
-    app.extensions = ['sympyprinting']
+app.extensions.append('sympy.interactive.ipythonprinting')
 

IPython/config/tests/test_configurable.py

@@ -27,7 +27,7 b' from IPython.config.configurable import ('
 )
 
 from IPython.utils.traitlets import (
-    Integer, Float, Unicode
+    Integer, Float, Unicode, List, Dict, Set,
 )
 
 from IPython.config.loader import Config
@@ -281,3 +281,80 b' class TestParentConfigurable(TestCase):'
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent2.MyParent.MyConfigurable.b)
 
+class Containers(Configurable):
+    lis = List(config=True)
+    def _lis_default(self):
+        return [-1]
+    
+    s = Set(config=True)
+    def _s_default(self):
+        return {'a'}
+    
+    d = Dict(config=True)
+    def _d_default(self):
+        return {'a' : 'b'}
+
+class TestConfigContainers(TestCase):
+    def test_extend(self):
+        c = Config()
+        c.Containers.lis.extend(range(5))
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, range(-1,5))
+
+    def test_insert(self):
+        c = Config()
+        c.Containers.lis.insert(0, 'a')
+        c.Containers.lis.insert(1, 'b')
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, ['a', 'b', -1])
+
+    def test_prepend(self):
+        c = Config()
+        c.Containers.lis.prepend([1,2])
+        c.Containers.lis.prepend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [2,3,1,2,-1])
+
+    def test_prepend_extend(self):
+        c = Config()
+        c.Containers.lis.prepend([1,2])
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [1,2,-1,2,3])
+
+    def test_append_extend(self):
+        c = Config()
+        c.Containers.lis.append([1,2])
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [-1,[1,2],2,3])
+
+    def test_extend_append(self):
+        c = Config()
+        c.Containers.lis.extend([2,3])
+        c.Containers.lis.append([1,2])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [-1,2,3,[1,2]])
+
+    def test_insert_extend(self):
+        c = Config()
+        c.Containers.lis.insert(0, 1)
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [1,-1,2,3])
+
+    def test_set_update(self):
+        c = Config()
+        c.Containers.s.update({0,1,2})
+        c.Containers.s.update({3})
+        obj = Containers(config=c)
+        self.assertEqual(obj.s, {'a', 0, 1, 2, 3})
+
+    def test_dict_update(self):
+        c = Config()
+        c.Containers.d.update({'c' : 'd'})
+        c.Containers.d.update({'e' : 'f'})
+        obj = Containers(config=c)
+        self.assertEqual(obj.d, {'a':'b', 'c':'d', 'e':'f'})
+    
+

IPython/config/tests/test_loader.py

@@ -9,7 +9,7 b' Authors:'
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2008 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -279,4 +279,14 b' class TestConfig(TestCase):'
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
         self.assertEqual(c1.Foo.baz, 2)
-        self.assertRaises(AttributeError, getattr, c2.Foo, 'baz')
+        self.assertNotIn('baz', c2.Foo)
+    
+    def test_contains(self):
+        c1 = Config({'Foo' : {'baz' : 2}})
+        c2 = Config({'Foo' : {'bar' : 1}})
+        self.assertIn('Foo', c1)
+        self.assertIn('Foo.baz', c1)
+        self.assertIn('Foo.bar', c2)
+        self.assertNotIn('Foo.bar', c1)
+    
+

IPython/core/application.py

@@ -282,10 +282,7 b' class BaseIPythonApplication(Application):'
         if self.profile_dir is not None:
             # already ran
             return
-        try:
-            # location explicitly specified:
-            location = self.config.ProfileDir.location
-        except AttributeError:
+        if 'ProfileDir.location' not in self.config:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
@@ -305,6 +302,7 b' class BaseIPythonApplication(Application):'
             else:
                 self.log.info("Using existing profile dir: %r"%p.location)
         else:
+            location = self.config.ProfileDir.location
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)

IPython/parallel/apps/ipcontrollerapp.py

@@ -399,9 +399,9 b' class IPControllerApp(BaseParallelApplication):'
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
-        try:
+        if 'TaskScheduler.scheme_name' in self.config:
             scheme = self.config.TaskScheduler.scheme_name
-        except AttributeError:
+        else:
             scheme = TaskScheduler.scheme_name.get_default_value()
         # Task Queue (in a Process)
         if scheme == 'pure':

IPython/parallel/apps/ipengineapp.py

@@ -211,14 +211,9 b' class IPEngineApp(BaseParallelApplication):'
         
         # allow hand-override of location for disambiguation
         # and ssh-server
-        try:
-            config.EngineFactory.location
-        except AttributeError:
+        if 'EngineFactory.location' not in config:
             config.EngineFactory.location = d['location']
-        
-        try:
-            config.EngineFactory.sshserver
-        except AttributeError:
+        if 'EngineFactory.sshserver' not in config:
             config.EngineFactory.sshserver = d.get('ssh')
         
         location = config.EngineFactory.location
@@ -313,21 +308,17 b' class IPEngineApp(BaseParallelApplication):'
             self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
             self.exit(1)
         
+        exec_lines = []
+        for app in ('IPKernelApp', 'InteractiveShellApp'):
+            if '%s.exec_lines' in config:
+                exec_lines = config.IPKernelApp.exec_lines = config[app].exec_lines
+                break
         
-        try:
-            exec_lines = config.IPKernelApp.exec_lines
-        except AttributeError:
-            try:
-                exec_lines = config.InteractiveShellApp.exec_lines
-            except AttributeError:
-                exec_lines = config.IPKernelApp.exec_lines = []
-        try:
-            exec_files = config.IPKernelApp.exec_files
-        except AttributeError:
-            try:
-                exec_files = config.InteractiveShellApp.exec_files
-            except AttributeError:
-                exec_files = config.IPKernelApp.exec_files = []
+        exec_files = []
+        for app in ('IPKernelApp', 'InteractiveShellApp'):
+            if '%s.exec_files' in config:
+                exec_files = config.IPKernelApp.exec_files = config[app].exec_files
+                break
         
         if self.startup_script:
             exec_files.append(self.startup_script)

IPython/parallel/controller/hub.py

@@ -255,10 +255,9 b' class HubFactory(RegistrationFactory):'
 
         ctx = self.context
         loop = self.loop
-
-        try:
+        if 'TaskScheduler.scheme_name' in self.config:
             scheme = self.config.TaskScheduler.scheme_name
-        except AttributeError:
+        else:
             from .scheduler import TaskScheduler
             scheme = TaskScheduler.scheme_name.get_default_value()
         

IPython/qt/console/qtconsoleapp.py

@@ -281,18 +281,10 b' class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp):'
         # are removed from the backend.
 
         # parse the colors arg down to current known labels
-        try:
-            colors = self.config.ZMQInteractiveShell.colors
-        except AttributeError:
-            colors = None
-        try:
-            style = self.config.IPythonWidget.syntax_style
-        except AttributeError:
-            style = None
-        try:
-            sheet = self.config.IPythonWidget.style_sheet
-        except AttributeError:
-            sheet = None
+        cfg = self.config
+        colors = cfg.ZMQInteractiveShell.colors if 'ZMQInteractiveShell.colors' in cfg else None
+        style = cfg.IPythonWidget.syntax_style if 'IPythonWidget.syntax_style' in cfg else None
+        sheet = cfg.IPythonWidget.style_sheet if 'IPythonWidget.style_sheet' in cfg else None
 
         # find the value for colors:
         if colors: