Merge pull request #4276 from minrk/config-extend

Support container methods in config

docs/source/whatsnew/pr/config-extend.rst

@@ -0,0 +1,15 b''
+Extending Configurable Containers
+---------------------------------
+
+Some configurable traits are containers (list, dict, set)
+Config objects now support calling ``extend``, ``update``, ``insert``, etc.
+on traits in config files, which will ultimately result in calling
+those methods on the original object.
+
+The effect being that you can now add to containers without having to copy/paste
+the initial value::
+
+    c = get_config()
+    c.InlineBackend.rc.update({ 'figure.figsize' : (6, 4) })
+
+

IPython/config/configurable.py

@@ -1,382 +1,387 b''
 # encoding: utf-8
 """
 A base class for objects that are configurable.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.config.configurable
    :parts: 3
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import datetime
 from copy import deepcopy
 
-from loader import Config
+from .loader import Config, LazyConfigValue
 from IPython.utils.traitlets import HasTraits, Instance
 from IPython.utils.text import indent, wrap_paragraphs
 
 
 #-----------------------------------------------------------------------------
 # Helper classes for Configurables
 #-----------------------------------------------------------------------------
 
 
 class ConfigurableError(Exception):
     pass
 
 
 class MultipleInstanceError(ConfigurableError):
     pass
 
 #-----------------------------------------------------------------------------
 # Configurable implementation
 #-----------------------------------------------------------------------------
 
 class Configurable(HasTraits):
 
     config = Instance(Config, (), {})
     parent = Instance('IPython.config.configurable.Configurable')
     created = None
 
     def __init__(self, **kwargs):
         """Create a configurable given a config config.
 
         Parameters
         ----------
         config : Config
             If this is empty, default values are used. If config is a
             :class:`Config` instance, it will be used to configure the
             instance.
         parent : Configurable instance, optional
             The parent Configurable instance of this object.
 
         Notes
         -----
         Subclasses of Configurable must call the :meth:`__init__` method of
         :class:`Configurable` *before* doing anything else and using
         :func:`super`::
 
             class MyConfigurable(Configurable):
                 def __init__(self, config=None):
                     super(MyConfigurable, self).__init__(config=config)
                     # Then any other code you need to finish initialization.
 
         This ensures that instances will be configured properly.
         """
         parent = kwargs.pop('parent', None)
         if parent is not None:
             # config is implied from parent
             if kwargs.get('config', None) is None:
                 kwargs['config'] = parent.config
             self.parent = parent
         
         config = kwargs.pop('config', None)
         if config is not None:
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
             # will share config. In fact, I did find such a side effect in
             # _config_changed below. If a config attribute value was a mutable type
             # all instances of a component were getting the same copy, effectively
             # making that a class attribute.
             # self.config = deepcopy(config)
             self.config = config
         # This should go second so individual keyword arguments override
         # the values in config.
         super(Configurable, self).__init__(**kwargs)
         self.created = datetime.datetime.now()
 
     #-------------------------------------------------------------------------
     # Static trait notifiations
     #-------------------------------------------------------------------------
     
     @classmethod
     def section_names(cls):
         """return section names as a list"""
         return  [c.__name__ for c in reversed(cls.__mro__) if
             issubclass(c, Configurable) and issubclass(cls, c)
         ]
     
     def _find_my_config(self, cfg):
         """extract my config from a global Config object
         
         will construct a Config object of only the config values that apply to me
         based on my mro(), as well as those of my parent(s) if they exist.
         
         If I am Bar and my parent is Foo, and their parent is Tim,
         this will return merge following config sections, in this order::
         
             [Bar, Foo.bar, Tim.Foo.Bar]
         
         With the last item being the highest priority.
         """
         cfgs = [cfg]
         if self.parent:
             cfgs.append(self.parent._find_my_config(cfg))
         my_config = Config()
         for c in cfgs:
             for sname in self.section_names():
                 # Don't do a blind getattr as that would cause the config to
                 # dynamically create the section with name Class.__name__.
                 if c._has_section(sname):
                     my_config.merge(c[sname])
         return my_config
-
+    
     def _load_config(self, cfg, section_names=None, traits=None):
         """load traits from a Config object"""
         
         if traits is None:
             traits = self.traits(config=True)
         if section_names is None:
             section_names = self.section_names()
         
         my_config = self._find_my_config(cfg)
         for name, config_value in my_config.iteritems():
             if name in traits:
+                if isinstance(config_value, LazyConfigValue):
+                    # ConfigValue is a wrapper for using append / update on containers
+                    # without having to copy the 
+                    initial = getattr(self, name)
+                    config_value = config_value.get_value(initial)
                 # We have to do a deepcopy here if we don't deepcopy the entire
                 # config object. If we don't, a mutable config_value will be
                 # shared by all instances, effectively making it a class attribute.
                 setattr(self, name, deepcopy(config_value))
 
     def _config_changed(self, name, old, new):
         """Update all the class traits having ``config=True`` as metadata.
 
         For any class trait with a ``config`` metadata attribute that is
         ``True``, we update the trait with the value of the corresponding
         config entry.
         """
         # Get all traits with a config metadata entry that is True
         traits = self.traits(config=True)
 
         # We auto-load config section for this class as well as any parent
         # classes that are Configurable subclasses.  This starts with Configurable
         # and works down the mro loading the config for each section.
         section_names = self.section_names()
         self._load_config(new, traits=traits, section_names=section_names)
 
     def update_config(self, config):
         """Fire the traits events when the config is updated."""
         # Save a copy of the current config.
         newconfig = deepcopy(self.config)
         # Merge the new config into the current one.
         newconfig.merge(config)
         # Save the combined config as self.config, which triggers the traits
         # events.
         self.config = newconfig
 
     @classmethod
     def class_get_help(cls, inst=None):
         """Get the help string for this class in ReST format.
         
         If `inst` is given, it's current trait values will be used in place of
         class defaults.
         """
         assert inst is None or isinstance(inst, cls)
         final_help = []
         final_help.append(u'%s options' % cls.__name__)
         final_help.append(len(final_help[0])*u'-')
         for k, v in sorted(cls.class_traits(config=True).iteritems()):
             help = cls.class_get_trait_help(v, inst)
             final_help.append(help)
         return '\n'.join(final_help)
 
     @classmethod
     def class_get_trait_help(cls, trait, inst=None):
         """Get the help string for a single trait.
         
         If `inst` is given, it's current trait values will be used in place of
         the class default.
         """
         assert inst is None or isinstance(inst, cls)
         lines = []
         header = "--%s.%s=<%s>" % (cls.__name__, trait.name, trait.__class__.__name__)
         lines.append(header)
         if inst is not None:
             lines.append(indent('Current: %r' % getattr(inst, trait.name), 4))
         else:
             try:
                 dvr = repr(trait.get_default_value())
             except Exception:
                 dvr = None # ignore defaults we can't construct
             if dvr is not None:
                 if len(dvr) > 64:
                     dvr = dvr[:61]+'...'
                 lines.append(indent('Default: %s' % dvr, 4))
         if 'Enum' in trait.__class__.__name__:
             # include Enum choices
             lines.append(indent('Choices: %r' % (trait.values,)))
 
         help = trait.get_metadata('help')
         if help is not None:
             help = '\n'.join(wrap_paragraphs(help, 76))
             lines.append(indent(help, 4))
         return '\n'.join(lines)
 
     @classmethod
     def class_print_help(cls, inst=None):
         """Get the help string for a single trait and print it."""
         print cls.class_get_help(inst)
 
     @classmethod
     def class_config_section(cls):
         """Get the config class config section"""
         def c(s):
             """return a commented, wrapped block."""
             s = '\n\n'.join(wrap_paragraphs(s, 78))
 
             return '# ' + s.replace('\n', '\n# ')
 
         # section header
         breaker = '#' + '-'*78
         s = "# %s configuration" % cls.__name__
         lines = [breaker, s, breaker, '']
         # get the description trait
         desc = cls.class_traits().get('description')
         if desc:
             desc = desc.default_value
         else:
             # no description trait, use __doc__
             desc = getattr(cls, '__doc__', '')
         if desc:
             lines.append(c(desc))
             lines.append('')
 
         parents = []
         for parent in cls.mro():
             # only include parents that are not base classes
             # and are not the class itself
             # and have some configurable traits to inherit
             if parent is not cls and issubclass(parent, Configurable) and \
                     parent.class_traits(config=True):
                 parents.append(parent)
 
         if parents:
             pstr = ', '.join([ p.__name__ for p in parents ])
             lines.append(c('%s will inherit config from: %s'%(cls.__name__, pstr)))
             lines.append('')
 
         for name, trait in cls.class_traits(config=True).iteritems():
             help = trait.get_metadata('help') or ''
             lines.append(c(help))
             lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))
             lines.append('')
         return '\n'.join(lines)
 
 
 
 class SingletonConfigurable(Configurable):
     """A configurable that only allows one instance.
 
     This class is for classes that should only have one instance of itself
     or *any* subclass. To create and retrieve such a class use the
     :meth:`SingletonConfigurable.instance` method.
     """
 
     _instance = None
 
     @classmethod
     def _walk_mro(cls):
         """Walk the cls.mro() for parent classes that are also singletons
 
         For use in instance()
         """
 
         for subclass in cls.mro():
             if issubclass(cls, subclass) and \
                     issubclass(subclass, SingletonConfigurable) and \
                     subclass != SingletonConfigurable:
                 yield subclass
 
     @classmethod
     def clear_instance(cls):
         """unset _instance for this class and singleton parents.
         """
         if not cls.initialized():
             return
         for subclass in cls._walk_mro():
             if isinstance(subclass._instance, cls):
                 # only clear instances that are instances
                 # of the calling class
                 subclass._instance = None
 
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global instance of this class.
 
         This method create a new instance if none have previously been created
         and returns a previously created instance is one already exists.
 
         The arguments and keyword arguments passed to this method are passed
         on to the :meth:`__init__` method of the class upon instantiation.
 
         Examples
         --------
 
         Create a singleton class using instance, and retrieve it::
 
             >>> from IPython.config.configurable import SingletonConfigurable
             >>> class Foo(SingletonConfigurable): pass
             >>> foo = Foo.instance()
             >>> foo == Foo.instance()
             True
 
         Create a subclass that is retrived using the base class instance::
 
             >>> class Bar(SingletonConfigurable): pass
             >>> class Bam(Bar): pass
             >>> bam = Bam.instance()
             >>> bam == Bar.instance()
             True
         """
         # Create and save the instance
         if cls._instance is None:
             inst = cls(*args, **kwargs)
             # Now make sure that the instance will also be returned by
             # parent classes' _instance attribute.
             for subclass in cls._walk_mro():
                 subclass._instance = inst
 
         if isinstance(cls._instance, cls):
             return cls._instance
         else:
             raise MultipleInstanceError(
                 'Multiple incompatible subclass instances of '
                 '%s are being created.' % cls.__name__
             )
 
     @classmethod
     def initialized(cls):
         """Has an instance been created?"""
         return hasattr(cls, "_instance") and cls._instance is not None
 
 
 class LoggingConfigurable(Configurable):
     """A parent class for Configurables that log.
 
     Subclasses have a log trait, and the default behavior
     is to get the logger from the currently running Application
     via Application.instance().log.
     """
 
     log = Instance('logging.Logger')
     def _log_default(self):
         from IPython.config.application import Application
         return Application.instance().log
 
 

IPython/config/loader.py

@@ -1,718 +1,821 b''
 """A simple configuration system.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.config.loader
    :parts: 3
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__ as builtin_mod
 import argparse
+import copy
 import os
 import re
 import sys
 
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat, warn
 from IPython.utils.encoding import DEFAULT_ENCODING
+from IPython.utils.traitlets import HasTraits, List, Any, TraitError
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 class ConfigLoaderError(ConfigError):
     pass
 
 class ConfigFileNotFound(ConfigError):
     pass
 
 class ArgumentError(ConfigLoaderError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
 
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
 
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
 
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
+class LazyConfigValue(HasTraits):
+    """Proxy object for exposing methods on configurable containers
+    
+    Exposes:
+    
+    - append, extend, insert on lists
+    - update on dicts
+    - update, add on sets
+    """
+    
+    _value = None
+    
+    # list methods
+    _extend = List()
+    _prepend = List()
+    
+    def append(self, obj):
+        self._extend.append(obj)
+    
+    def extend(self, other):
+        self._extend.extend(other)
+    
+    def prepend(self, other):
+        """like list.extend, but for the front"""
+        self._prepend[:0] = other
+    
+    _inserts = List()
+    def insert(self, index, other):
+        if not isinstance(index, int):
+            raise TypeError("An integer is required")
+        self._inserts.append((index, other))
+    
+    # dict methods
+    # update is used for both dict and set
+    _update = Any()
+    def update(self, other):
+        if self._update is None:
+            if isinstance(other, dict):
+                self._update = {}
+            else:
+                self._update = set()
+        self._update.update(other)
+    
+    # set methods
+    def add(self, obj):
+        self.update({obj})
+    
+    def get_value(self, initial):
+        """construct the value from the initial one
+        
+        after applying any insert / extend / update changes
+        """
+        if self._value is not None:
+            return self._value
+        value = copy.deepcopy(initial)
+        if isinstance(value, list):
+            for idx, obj in self._inserts:
+                value.insert(idx, obj)
+            value[:0] = self._prepend
+            value.extend(self._extend)
+        
+        elif isinstance(value, dict):
+            if self._update:
+                value.update(self._update)
+        elif isinstance(value, set):
+            if self._update:
+                value.update(self._update)
+        self._value = value
+        return value
+    
+    def to_dict(self):
+        """return JSONable dict form of my data
+        
+        Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
+        """
+        d = {}
+        if self._update:
+            d['update'] = self._update
+        if self._extend:
+            d['extend'] = self._extend
+        if self._prepend:
+            d['prepend'] = self._prepend
+        elif self._inserts:
+            d['inserts'] = self._inserts
+        return d
+
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
         self._ensure_subconfig()
     
     def _ensure_subconfig(self):
         """ensure that sub-dicts that should be Config objects are
         
         casts dicts that are under section keys to Config objects,
         which is necessary for constructing Config objects from dict literals.
         """
         for key in self:
             obj = self[key]
             if self._is_section_key(key) \
                     and isinstance(obj, dict) \
                     and not isinstance(obj, Config):
                 dict.__setattr__(self, key, Config(obj))
     
     def _merge(self, other):
         """deprecated alias, use Config.merge()"""
         self.merge(other)
     
     def merge(self, other):
         """merge another config object into this one"""
         to_update = {}
         for k, v in other.iteritems():
             if k not in self:
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config) and isinstance(self[k], Config):
                     # Recursively merge common sub Configs
                     self[k].merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
     def __contains__(self, key):
+        # allow nested contains of the form `"Section.key" in config`
+        if '.' in key:
+            first, remainder = key.split('.', 1)
+            if first not in self:
+                return False
+            return remainder in self[first]
+        
+        # we always have Sections
         if self._is_section_key(key):
             return True
         else:
             return super(Config, self).__contains__(key)
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
 
     def _has_section(self, key):
         if self._is_section_key(key):
             if super(Config, self).__contains__(key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
-
+    
     def __getitem__(self, key):
         # We cannot use directly self._is_section_key, because it triggers
         # infinite recursion on top of PyPy. Instead, we manually fish the
         # bound method.
         is_section_key = self.__class__._is_section_key.__get__(self)
 
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are
         # builtins.
         try:
             return getattr(builtin_mod, key)
         except AttributeError:
             pass
         if is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
-            return dict.__getitem__(self, key)
+            try:
+                return dict.__getitem__(self, key)
+            except KeyError:
+                # undefined
+                v = LazyConfigValue()
+                dict.__setitem__(self, key, v)
+                return v
+            
 
     def __setitem__(self, key, value):
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError as e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError as e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError as e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
 
     The resulting configuration is packaged as a :class:`Struct`.
 
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
 
         Examples
         --------
 
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
 
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
 
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         try:
             self._find_file()
         except IOError as e:
             raise ConfigFileNotFound(str(e))
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  It allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
 
         # If a profile is specified, the config file will be loaded
         # from that profile
 
         def load_subconfig(fname, profile=None):
             # import here to prevent circular imports
             from IPython.core.profiledir import ProfileDir, ProfileDirError
             if profile is not None:
                 try:
                     profile_dir = ProfileDir.find_profile_dir_by_name(
                             get_ipython_dir(),
                             profile,
                     )
                 except ProfileDirError:
                     return
                 path = profile_dir.location
             else:
                 path = self.path
             loader = PyFileConfigLoader(fname, path)
             try:
                 sub_config = loader.load_config()
             except ConfigFileNotFound:
                 # Pass silently if the sub config is not there. This happens
                 # when a user s using a profile, but not the default config.
                 pass
             else:
                 self.config.merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(
             load_subconfig=load_subconfig,
             get_config=get_config,
             __file__=self.full_filename,
         )
         fs_encoding = sys.getfilesystemencoding() or 'ascii'
         conf_filename = self.full_filename.encode(fs_encoding)
         py3compat.execfile(conf_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
     def _exec_config_str(self, lhs, rhs):
         """execute self.config.<lhs> = <rhs>
         
         * expands ~ with expanduser
         * tries to assign with raw eval, otherwise assigns with just the string,
           allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent.  *Not*
           equivalent are `--C.a=4` and `--C.a='4'`.
         """
         rhs = os.path.expanduser(rhs)
         try:
             # Try to see if regular Python syntax will work. This
             # won't handle strings as the quote marks are removed
             # by the system shell.
             value = eval(rhs)
         except (NameError, SyntaxError):
             # This case happens if the rhs is a string.
             value = rhs
 
         exec u'self.config.%s = value' % lhs
 
     def _load_flag(self, cfg):
         """update self.config from a flag, which can be a dict or Config"""
         if isinstance(cfg, (dict, Config)):
             # don't clobber whole config sections, update
             # each section from config:
             for sec,c in cfg.iteritems():
                 self.config[sec].update(c)
         else:
             raise TypeError("Invalid flag: %r" % cfg)
 
 # raw --identifier=value pattern
 # but *also* accept '-' as wordsep, for aliases
 # accepts:  --foo=a
 #           --Class.trait=value
 #           --alias-name=value
 # rejects:  -foo=value
 #           --foo
 #           --Class.trait
 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
 
 # just flags, no assignments, with two *or one* leading '-'
 # accepts:  --foo
 #           -foo-bar-again
 # rejects:  --anything=anything
 #           --two.word
 
 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
 
 class KeyValueConfigLoader(CommandLineConfigLoader):
     """A config loader that loads key value pairs from the command line.
 
     This allows command line options to be gives in the following form::
 
         ipython --profile="foo" --InteractiveShell.autocall=False
     """
 
     def __init__(self, argv=None, aliases=None, flags=None):
         """Create a key value pair config loader.
 
         Parameters
         ----------
         argv : list
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then sys.argv[1:] will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Vaues can be Config objects,
             dicts, or "key=value" strings.  If Config or dict, when the flag
             is triggered, The flag is loaded as `self.config.update(m)`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
 
         Examples
         --------
 
             >>> from IPython.config.loader import KeyValueConfigLoader
             >>> cl = KeyValueConfigLoader()
             >>> d = cl.load_config(["--A.name='brian'","--B.number=0"])
             >>> sorted(d.items())
             [('A', {'name': 'brian'}), ('B', {'number': 0})]
         """
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
 
     def clear(self):
         super(KeyValueConfigLoader, self).clear()
         self.extra_args = []
 
 
     def _decode_argv(self, argv, enc=None):
         """decode argv if bytes, using stin.encoding, falling back on default enc"""
         uargv = []
         if enc is None:
             enc = DEFAULT_ENCODING
         for arg in argv:
             if not isinstance(arg, unicode):
                 # only decode if not already decoded
                 arg = arg.decode(enc)
             uargv.append(arg)
         return uargv
 
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse the configuration and generate the Config object.
 
         After loading, any arguments that are not key-value or
         flags will be stored in self.extra_args - a list of
         unparsed command-line arguments.  This is used for
         arguments such as input files or subcommands.
 
         Parameters
         ----------
         argv : list, optional
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then self.argv will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Values can be Config objects
             or dicts.  When the flag is triggered, The config is loaded as
             `self.config.update(cfg)`.
         """
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
 
         # ensure argv is a list of unicode strings:
         uargv = self._decode_argv(argv)
         for idx,raw in enumerate(uargv):
             # strip leading '-'
             item = raw.lstrip('-')
 
             if raw == '--':
                 # don't parse arguments after '--'
                 # this is useful for relaying arguments to scripts, e.g.
                 # ipython -i foo.py --matplotlib=qt -- args after '--' go-to-foo.py
                 self.extra_args.extend(uargv[idx+1:])
                 break
 
             if kv_pattern.match(raw):
                 lhs,rhs = item.split('=',1)
                 # Substitute longnames for aliases.
                 if lhs in aliases:
                     lhs = aliases[lhs]
                 if '.' not in lhs:
                     # probably a mistyped alias, but not technically illegal
                     warn.warn("Unrecognized alias: '%s', it will probably have no effect."%lhs)
                 try:
                     self._exec_config_str(lhs, rhs)
                 except Exception:
                     raise ArgumentError("Invalid argument: '%s'" % raw)
 
             elif flag_pattern.match(raw):
                 if item in flags:
                     cfg,help = flags[item]
                     self._load_flag(cfg)
                 else:
                     raise ArgumentError("Unrecognized flag: '%s'"%raw)
             elif raw.startswith('-'):
                 kv = '--'+item
                 if kv_pattern.match(kv):
                     raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
                 else:
                     raise ArgumentError("Invalid argument: '%s'"%raw)
             else:
                 # keep all args that aren't valid in a list,
                 # in case our parent knows what to do with them.
                 self.extra_args.append(item)
         return self.config
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
     """A loader that uses the argparse module to load from the command line."""
 
     def __init__(self, argv=None, aliases=None, flags=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
         """
         super(CommandLineConfigLoader, self).__init__()
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
         self.parser_args = parser_args
         self.version = parser_kw.pop("version", None)
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse command line arguments and return as a Config object.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         self._create_parser(aliases, flags)
         self._parse_args(argv)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self, aliases=None, flags=None):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments(aliases, flags)
 
     def _add_arguments(self, aliases=None, flags=None):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data"""
         # decode sys.argv to support unicode command-line options
         enc = DEFAULT_ENCODING
         uargs = [py3compat.cast_unicode(a, enc) for a in args]
         self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).iteritems():
             exec "self.config.%s = v"%k in locals(), globals()
 
 class KVArgParseConfigLoader(ArgParseConfigLoader):
     """A config loader that loads aliases and flags with argparse,
     but will use KVLoader for the rest.  This allows better parsing
     of common args, such as `ipython -c 'print 5'`, but still gets
     arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
 
     def _add_arguments(self, aliases=None, flags=None):
         self.alias_flags = {}
         # print aliases, flags
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         paa = self.parser.add_argument
         for key,value in aliases.iteritems():
             if key in flags:
                 # flags
                 nargs = '?'
             else:
                 nargs = None
             if len(key) is 1:
                 paa('-'+key, '--'+key, type=unicode, dest=value, nargs=nargs)
             else:
                 paa('--'+key, type=unicode, dest=value, nargs=nargs)
         for key, (value, help) in flags.iteritems():
             if key in self.aliases:
                 #
                 self.alias_flags[self.aliases[key]] = value
                 continue
             if len(key) is 1:
                 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
             else:
                 paa('--'+key, action='append_const', dest='_flags', const=value)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
         # remove subconfigs list from namespace before transforming the Namespace
         if '_flags' in self.parsed_data:
             subcs = self.parsed_data._flags
             del self.parsed_data._flags
         else:
             subcs = []
 
         for k, v in vars(self.parsed_data).iteritems():
             if v is None:
                 # it was a flag that shares the name of an alias
                 subcs.append(self.alias_flags[k])
             else:
                 # eval the KV assignment
                 self._exec_config_str(k, v)
 
         for subc in subcs:
             self._load_flag(subc)
 
         if self.extra_args:
             sub_parser = KeyValueConfigLoader()
             sub_parser.load_config(self.extra_args)
             self.config.merge(sub_parser.config)
             self.extra_args = sub_parser.extra_args
 
 
 def load_pyconfig_files(config_files, path):
     """Load multiple Python config files, merging each of them in turn.
 
     Parameters
     ==========
     config_files : list of str
         List of config files names to load and merge into the config.
     path : unicode
         The full path to the location of the config files.
     """
     config = Config()
     for cf in config_files:
         loader = PyFileConfigLoader(cf, path=path)
         try:
             next_config = loader.load_config()
         except ConfigFileNotFound:
             pass
         except:
             raise
         else:
             config.merge(next_config)
     return config

IPython/config/profile/cluster/ipython_config.py

@@ -1,19 +1,13 b''
 c = get_config()
 app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
 load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 from IPython.parallel import *
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)
 

IPython/config/profile/math/ipython_config.py

@@ -1,21 +1,13 b''
 c = get_config()
 app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
 load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 import cmath
 from math import *
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
-
+app.exec_lines.append(lines)

IPython/config/profile/pysh/ipython_config.py

@@ -1,30 +1,24 b''
 c = get_config()
 app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
 load_subconfig('ipython_config.py', profile='default')
 
 c.PromptManager.in_template = r'{color.LightGreen}\u@\h{color.LightBlue}[{color.LightCyan}\Y1{color.LightBlue}]{color.Green}|\#> '
 c.PromptManager.in2_template = r'{color.Green}|{color.LightGreen}\D{color.Green}> '
 c.PromptManager.out_template = r'<\#> '
 
 c.PromptManager.justify = True
 
 c.InteractiveShell.separate_in = ''
 c.InteractiveShell.separate_out = ''
 c.InteractiveShell.separate_out2 = ''
 
 c.PrefilterManager.multi_line_specials = True
 
 lines = """
 %rehashx
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)

IPython/config/profile/sympy/ipython_config.py

@@ -1,30 +1,20 b''
 c = get_config()
 app = c.InteractiveShellApp
 
 # This can be used at any point in a config file to load a sub config
 # and merge it into the current one.
 load_subconfig('ipython_config.py', profile='default')
 
 lines = """
 from __future__ import division
 from sympy import *
 x, y, z, t = symbols('x y z t')
 k, m, n = symbols('k m n', integer=True)
 f, g, h = symbols('f g h', cls=Function)
 """
 
-# You have to make sure that attributes that are containers already
-# exist before using them.  Simple assigning a new list will override
-# all previous values.
-
-if hasattr(app, 'exec_lines'):
-    app.exec_lines.append(lines)
-else:
-    app.exec_lines = [lines]
+app.exec_lines.append(lines)
 
 # Load the sympy_printing extension to enable nice printing of sympy expr's.
-if hasattr(app, 'extensions'):
-    app.extensions.append('sympyprinting')
-else:
-    app.extensions = ['sympyprinting']
+app.extensions.append('sympy.interactive.ipythonprinting')
 

IPython/config/tests/test_configurable.py

@@ -1,283 +1,360 b''
 # encoding: utf-8
 """
 Tests for IPython.config.configurable
 
 Authors:
 
 * Brian Granger
 * Fernando Perez (design help)
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from unittest import TestCase
 
 from IPython.config.configurable import (
     Configurable, 
     SingletonConfigurable
 )
 
 from IPython.utils.traitlets import (
-    Integer, Float, Unicode
+    Integer, Float, Unicode, List, Dict, Set,
 )
 
 from IPython.config.loader import Config
 from IPython.utils.py3compat import PY3
 
 #-----------------------------------------------------------------------------
 # Test cases
 #-----------------------------------------------------------------------------
 
 
 class MyConfigurable(Configurable):
     a = Integer(1, config=True, help="The integer a.")
     b = Float(1.0, config=True, help="The integer b.")
     c = Unicode('no config')
 
 
 mc_help=u"""MyConfigurable options
 ----------------------
 --MyConfigurable.a=<Integer>
     Default: 1
     The integer a.
 --MyConfigurable.b=<Float>
     Default: 1.0
     The integer b."""
 
 mc_help_inst=u"""MyConfigurable options
 ----------------------
 --MyConfigurable.a=<Integer>
     Current: 5
     The integer a.
 --MyConfigurable.b=<Float>
     Current: 4.0
     The integer b."""
 
 # On Python 3, the Integer trait is a synonym for Int
 if PY3:
     mc_help = mc_help.replace(u"<Integer>", u"<Int>")
     mc_help_inst = mc_help_inst.replace(u"<Integer>", u"<Int>")
 
 class Foo(Configurable):
     a = Integer(0, config=True, help="The integer a.")
     b = Unicode('nope', config=True)
 
 
 class Bar(Foo):
     b = Unicode('gotit', config=False, help="The string b.")
     c = Float(config=True, help="The string c.")
 
 
 class TestConfigurable(TestCase):
 
     def test_default(self):
         c1 = Configurable()
         c2 = Configurable(config=c1.config)
         c3 = Configurable(config=c2.config)
         self.assertEqual(c1.config, c2.config)
         self.assertEqual(c2.config, c3.config)
 
     def test_custom(self):
         config = Config()
         config.foo = 'foo'
         config.bar = 'bar'
         c1 = Configurable(config=config)
         c2 = Configurable(config=c1.config)
         c3 = Configurable(config=c2.config)
         self.assertEqual(c1.config, config)
         self.assertEqual(c2.config, config)
         self.assertEqual(c3.config, config)
         # Test that copies are not made
         self.assertTrue(c1.config is config)
         self.assertTrue(c2.config is config)
         self.assertTrue(c3.config is config)
         self.assertTrue(c1.config is c2.config)
         self.assertTrue(c2.config is c3.config)
         
     def test_inheritance(self):
         config = Config()
         config.MyConfigurable.a = 2
         config.MyConfigurable.b = 2.0
         c1 = MyConfigurable(config=config)
         c2 = MyConfigurable(config=c1.config)
         self.assertEqual(c1.a, config.MyConfigurable.a)
         self.assertEqual(c1.b, config.MyConfigurable.b)
         self.assertEqual(c2.a, config.MyConfigurable.a)
         self.assertEqual(c2.b, config.MyConfigurable.b)
 
     def test_parent(self):
         config = Config()
         config.Foo.a = 10
         config.Foo.b = "wow"
         config.Bar.b = 'later'
         config.Bar.c = 100.0
         f = Foo(config=config)
         b = Bar(config=f.config)
         self.assertEqual(f.a, 10)
         self.assertEqual(f.b, 'wow')
         self.assertEqual(b.b, 'gotit')
         self.assertEqual(b.c, 100.0)
 
     def test_override1(self):
         config = Config()
         config.MyConfigurable.a = 2
         config.MyConfigurable.b = 2.0
         c = MyConfigurable(a=3, config=config)
         self.assertEqual(c.a, 3)
         self.assertEqual(c.b, config.MyConfigurable.b)
         self.assertEqual(c.c, 'no config')
 
     def test_override2(self):
         config = Config()
         config.Foo.a = 1
         config.Bar.b = 'or'  # Up above b is config=False, so this won't do it.
         config.Bar.c = 10.0
         c = Bar(config=config)
         self.assertEqual(c.a, config.Foo.a)
         self.assertEqual(c.b, 'gotit')
         self.assertEqual(c.c, config.Bar.c)
         c = Bar(a=2, b='and', c=20.0, config=config)
         self.assertEqual(c.a, 2)
         self.assertEqual(c.b, 'and')
         self.assertEqual(c.c, 20.0)
 
     def test_help(self):
         self.assertEqual(MyConfigurable.class_get_help(), mc_help)
 
     def test_help_inst(self):
         inst = MyConfigurable(a=5, b=4)
         self.assertEqual(MyConfigurable.class_get_help(inst), mc_help_inst)
 
 
 class TestSingletonConfigurable(TestCase):
 
     def test_instance(self):
         class Foo(SingletonConfigurable): pass
         self.assertEqual(Foo.initialized(), False)
         foo = Foo.instance()
         self.assertEqual(Foo.initialized(), True)
         self.assertEqual(foo, Foo.instance())
         self.assertEqual(SingletonConfigurable._instance, None)
 
     def test_inheritance(self):
         class Bar(SingletonConfigurable): pass
         class Bam(Bar): pass
         self.assertEqual(Bar.initialized(), False)
         self.assertEqual(Bam.initialized(), False)
         bam = Bam.instance()
         bam == Bar.instance()
         self.assertEqual(Bar.initialized(), True)
         self.assertEqual(Bam.initialized(), True)
         self.assertEqual(bam, Bam._instance)
         self.assertEqual(bam, Bar._instance)
         self.assertEqual(SingletonConfigurable._instance, None)
 
 
 class MyParent(Configurable):
     pass
 
 class MyParent2(MyParent):
     pass
 
 class TestParentConfigurable(TestCase):
     
     def test_parent_config(self):
         cfg = Config({
             'MyParent' : {
                 'MyConfigurable' : {
                     'b' : 2.0,
                 }
             }
         })
         parent = MyParent(config=cfg)
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent.MyConfigurable.b)
 
     def test_parent_inheritance(self):
         cfg = Config({
             'MyParent' : {
                 'MyConfigurable' : {
                     'b' : 2.0,
                 }
             }
         })
         parent = MyParent2(config=cfg)
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent.MyConfigurable.b)
 
     def test_multi_parent(self):
         cfg = Config({
             'MyParent2' : {
                 'MyParent' : {
                     'MyConfigurable' : {
                         'b' : 2.0,
                     }
                 },
                 # this one shouldn't count
                 'MyConfigurable' : {
                     'b' : 3.0,
                 },
             }
         })
         parent2 = MyParent2(config=cfg)
         parent = MyParent(parent=parent2)
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent2.MyParent.MyConfigurable.b)
 
     def test_parent_priority(self):
         cfg = Config({
             'MyConfigurable' : {
                 'b' : 2.0,
             },
             'MyParent' : {
                 'MyConfigurable' : {
                     'b' : 3.0,
                 }
             },
             'MyParent2' : {
                 'MyConfigurable' : {
                     'b' : 4.0,
                 }
             }
         })
         parent = MyParent2(config=cfg)
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent2.MyConfigurable.b)
 
     def test_multi_parent_priority(self):
         cfg = Config({
             'MyConfigurable' : {
                 'b' : 2.0,
             },
             'MyParent' : {
                 'MyConfigurable' : {
                     'b' : 3.0,
                 }
             },
             'MyParent2' : {
                 'MyConfigurable' : {
                     'b' : 4.0,
                 }
             },
             'MyParent2' : {
                 'MyParent' : {
                     'MyConfigurable' : {
                         'b' : 5.0,
                     }
                 }
             }
         })
         parent2 = MyParent2(config=cfg)
         parent = MyParent2(parent=parent2)
         myc = MyConfigurable(parent=parent)
         self.assertEqual(myc.b, parent.config.MyParent2.MyParent.MyConfigurable.b)
 
+class Containers(Configurable):
+    lis = List(config=True)
+    def _lis_default(self):
+        return [-1]
+    
+    s = Set(config=True)
+    def _s_default(self):
+        return {'a'}
+    
+    d = Dict(config=True)
+    def _d_default(self):
+        return {'a' : 'b'}
+
+class TestConfigContainers(TestCase):
+    def test_extend(self):
+        c = Config()
+        c.Containers.lis.extend(range(5))
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, range(-1,5))
+
+    def test_insert(self):
+        c = Config()
+        c.Containers.lis.insert(0, 'a')
+        c.Containers.lis.insert(1, 'b')
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, ['a', 'b', -1])
+
+    def test_prepend(self):
+        c = Config()
+        c.Containers.lis.prepend([1,2])
+        c.Containers.lis.prepend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [2,3,1,2,-1])
+
+    def test_prepend_extend(self):
+        c = Config()
+        c.Containers.lis.prepend([1,2])
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [1,2,-1,2,3])
+
+    def test_append_extend(self):
+        c = Config()
+        c.Containers.lis.append([1,2])
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [-1,[1,2],2,3])
+
+    def test_extend_append(self):
+        c = Config()
+        c.Containers.lis.extend([2,3])
+        c.Containers.lis.append([1,2])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [-1,2,3,[1,2]])
+
+    def test_insert_extend(self):
+        c = Config()
+        c.Containers.lis.insert(0, 1)
+        c.Containers.lis.extend([2,3])
+        obj = Containers(config=c)
+        self.assertEqual(obj.lis, [1,-1,2,3])
+
+    def test_set_update(self):
+        c = Config()
+        c.Containers.s.update({0,1,2})
+        c.Containers.s.update({3})
+        obj = Containers(config=c)
+        self.assertEqual(obj.s, {'a', 0, 1, 2, 3})
+
+    def test_dict_update(self):
+        c = Config()
+        c.Containers.d.update({'c' : 'd'})
+        c.Containers.d.update({'e' : 'f'})
+        obj = Containers(config=c)
+        self.assertEqual(obj.d, {'a':'b', 'c':'d', 'e':'f'})
+    
+

IPython/config/tests/test_loader.py

@@ -1,282 +1,292 b''
 # encoding: utf-8
 """
 Tests for IPython.config.loader
 
 Authors:
 
 * Brian Granger
 * Fernando Perez (design help)
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2008 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 from tempfile import mkstemp
 from unittest import TestCase
 
 from nose import SkipTest
 
 from IPython.testing.tools import mute_warn
 
 from IPython.config.loader import (
     Config,
     PyFileConfigLoader,
     KeyValueConfigLoader,
     ArgParseConfigLoader,
     KVArgParseConfigLoader,
     ConfigError
 )
 
 #-----------------------------------------------------------------------------
 # Actual tests
 #-----------------------------------------------------------------------------
 
 
 pyfile = """
 c = get_config()
 c.a=10
 c.b=20
 c.Foo.Bar.value=10
 c.Foo.Bam.value=list(range(10))  # list() is just so it's the same on Python 3
 c.D.C.value='hi there'
 """
 
 class TestPyFileCL(TestCase):
 
     def test_basic(self):
         fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
         # Unlink the file
         cl = PyFileConfigLoader(fname)
         config = cl.load_config()
         self.assertEqual(config.a, 10)
         self.assertEqual(config.b, 20)
         self.assertEqual(config.Foo.Bar.value, 10)
         self.assertEqual(config.Foo.Bam.value, range(10))
         self.assertEqual(config.D.C.value, 'hi there')
 
 class MyLoader1(ArgParseConfigLoader):
     def _add_arguments(self, aliases=None, flags=None):
         p = self.parser
         p.add_argument('-f', '--foo', dest='Global.foo', type=str)
         p.add_argument('-b', dest='MyClass.bar', type=int)
         p.add_argument('-n', dest='n', action='store_true')
         p.add_argument('Global.bam', type=str)
 
 class MyLoader2(ArgParseConfigLoader):
     def _add_arguments(self, aliases=None, flags=None):
         subparsers = self.parser.add_subparsers(dest='subparser_name')
         subparser1 = subparsers.add_parser('1')
         subparser1.add_argument('-x',dest='Global.x')
         subparser2 = subparsers.add_parser('2')
         subparser2.add_argument('y')
 
 class TestArgParseCL(TestCase):
 
     def test_basic(self):
         cl = MyLoader1()
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEqual(config.Global.foo, 'hi')
         self.assertEqual(config.MyClass.bar, 10)
         self.assertEqual(config.n, True)
         self.assertEqual(config.Global.bam, 'wow')
         config = cl.load_config(['wow'])
         self.assertEqual(config.keys(), ['Global'])
         self.assertEqual(config.Global.keys(), ['bam'])
         self.assertEqual(config.Global.bam, 'wow')
 
     def test_add_arguments(self):
         cl = MyLoader2()
         config = cl.load_config('2 frobble'.split())
         self.assertEqual(config.subparser_name, '2')
         self.assertEqual(config.y, 'frobble')
         config = cl.load_config('1 -x frobble'.split())
         self.assertEqual(config.subparser_name, '1')
         self.assertEqual(config.Global.x, 'frobble')
 
     def test_argv(self):
         cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
         config = cl.load_config()
         self.assertEqual(config.Global.foo, 'hi')
         self.assertEqual(config.MyClass.bar, 10)
         self.assertEqual(config.n, True)
         self.assertEqual(config.Global.bam, 'wow')
 
 
 class TestKeyValueCL(TestCase):
     klass = KeyValueConfigLoader
 
     def test_basic(self):
         cl = self.klass()
         argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
         with mute_warn():
             config = cl.load_config(argv)
         self.assertEqual(config.a, 10)
         self.assertEqual(config.b, 20)
         self.assertEqual(config.Foo.Bar.value, 10)
         self.assertEqual(config.Foo.Bam.value, range(10))
         self.assertEqual(config.D.C.value, 'hi there')
     
     def test_expanduser(self):
         cl = self.klass()
         argv = ['--a=~/1/2/3', '--b=~', '--c=~/', '--d="~/"']
         with mute_warn():
             config = cl.load_config(argv)
         self.assertEqual(config.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.b, os.path.expanduser('~'))
         self.assertEqual(config.c, os.path.expanduser('~/'))
         self.assertEqual(config.d, '~/')
     
     def test_extra_args(self):
         cl = self.klass()
         with mute_warn():
             config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
         self.assertEqual(cl.extra_args, ['b', 'd'])
         self.assertEqual(config.a, 5)
         self.assertEqual(config.c, 10)
         with mute_warn():
             config = cl.load_config(['--', '--a=5', '--c=10'])
         self.assertEqual(cl.extra_args, ['--a=5', '--c=10'])
     
     def test_unicode_args(self):
         cl = self.klass()
         argv = [u'--a=épsîlön']
         with mute_warn():
             config = cl.load_config(argv)
         self.assertEqual(config.a, u'épsîlön')
     
     def test_unicode_bytes_args(self):
         uarg = u'--a=é'
         try:
             barg = uarg.encode(sys.stdin.encoding)
         except (TypeError, UnicodeEncodeError):
             raise SkipTest("sys.stdin.encoding can't handle 'é'")
         
         cl = self.klass()
         with mute_warn():
             config = cl.load_config([barg])
         self.assertEqual(config.a, u'é')
     
     def test_unicode_alias(self):
         cl = self.klass()
         argv = [u'--a=épsîlön']
         with mute_warn():
             config = cl.load_config(argv, aliases=dict(a='A.a'))
         self.assertEqual(config.A.a, u'épsîlön')
 
 
 class TestArgParseKVCL(TestKeyValueCL):
     klass = KVArgParseConfigLoader
 
     def test_expanduser2(self):
         cl = self.klass()
         argv = ['-a', '~/1/2/3', '--b', "'~/1/2/3'"]
         with mute_warn():
             config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b'))
         self.assertEqual(config.A.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.A.b, '~/1/2/3')
     
     def test_eval(self):
         cl = self.klass()
         argv = ['-c', 'a=5']
         with mute_warn():
             config = cl.load_config(argv, aliases=dict(c='A.c'))
         self.assertEqual(config.A.c, u"a=5")
     
 
 class TestConfig(TestCase):
 
     def test_setget(self):
         c = Config()
         c.a = 10
         self.assertEqual(c.a, 10)
         self.assertEqual('b' in c, False)
 
     def test_auto_section(self):
         c = Config()
         self.assertEqual('A' in c, True)
         self.assertEqual(c._has_section('A'), False)
         A = c.A
         A.foo = 'hi there'
         self.assertEqual(c._has_section('A'), True)
         self.assertEqual(c.A.foo, 'hi there')
         del c.A
         self.assertEqual(len(c.A.keys()),0)
 
     def test_merge_doesnt_exist(self):
         c1 = Config()
         c2 = Config()
         c2.bar = 10
         c2.Foo.bar = 10
         c1.merge(c2)
         self.assertEqual(c1.Foo.bar, 10)
         self.assertEqual(c1.bar, 10)
         c2.Bar.bar = 10
         c1.merge(c2)
         self.assertEqual(c1.Bar.bar, 10)
 
     def test_merge_exists(self):
         c1 = Config()
         c2 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c2.Foo.bar = 20
         c2.Foo.wow = 40
         c1.merge(c2)
         self.assertEqual(c1.Foo.bam, 30)
         self.assertEqual(c1.Foo.bar, 20)
         self.assertEqual(c1.Foo.wow, 40)
         c2.Foo.Bam.bam = 10
         c1.merge(c2)
         self.assertEqual(c1.Foo.Bam.bam, 10)
 
     def test_deepcopy(self):
         c1 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c1.a = 'asdf'
         c1.b = range(10)
         import copy
         c2 = copy.deepcopy(c1)
         self.assertEqual(c1, c2)
         self.assertTrue(c1 is not c2)
         self.assertTrue(c1.Foo is not c2.Foo)
 
     def test_builtin(self):
         c1 = Config()
         exec 'foo = True' in c1
         self.assertEqual(c1.foo, True)
         c1.format = "json"
     
     def test_fromdict(self):
         c1 = Config({'Foo' : {'bar' : 1}})
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
     
     def test_fromdictmerge(self):
         c1 = Config()
         c2 = Config({'Foo' : {'bar' : 1}})
         c1.merge(c2)
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
 
     def test_fromdictmerge2(self):
         c1 = Config({'Foo' : {'baz' : 2}})
         c2 = Config({'Foo' : {'bar' : 1}})
         c1.merge(c2)
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
         self.assertEqual(c1.Foo.baz, 2)
-        self.assertRaises(AttributeError, getattr, c2.Foo, 'baz')
+        self.assertNotIn('baz', c2.Foo)
+    
+    def test_contains(self):
+        c1 = Config({'Foo' : {'baz' : 2}})
+        c2 = Config({'Foo' : {'bar' : 1}})
+        self.assertIn('Foo', c1)
+        self.assertIn('Foo.baz', c1)
+        self.assertIn('Foo.bar', c2)
+        self.assertNotIn('Foo.bar', c1)
+    
+

IPython/core/application.py

@@ -1,383 +1,381 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating configurables.
 
 The job of an :class:`Application` is to create the master configuration
 object and then create the configurable objects, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import atexit
 import errno
 import glob
 import logging
 import os
 import shutil
 import sys
 
 from IPython.config.application import Application, catch_config_error
 from IPython.config.loader import ConfigFileNotFound
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instance
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Base Application Class
 #-----------------------------------------------------------------------------
 
 # aliases and flags
 
 base_aliases = {
     'profile-dir' : 'ProfileDir.location',
     'profile' : 'BaseIPythonApplication.profile',
     'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
     'log-level' : 'Application.log_level',
     'config' : 'BaseIPythonApplication.extra_config_file',
 }
 
 base_flags = dict(
     debug = ({'Application' : {'log_level' : logging.DEBUG}},
             "set log level to logging.DEBUG (maximize logging output)"),
     quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
             "set log level to logging.CRITICAL (minimize logging output)"),
     init = ({'BaseIPythonApplication' : {
                     'copy_config_files' : True,
                     'auto_create' : True}
             }, """Initialize profile with default config files.  This is equivalent
             to running `ipython profile create <profile>` prior to startup.
             """)
 )
 
 
 class BaseIPythonApplication(Application):
 
     name = Unicode(u'ipython')
     description = Unicode(u'IPython: an enhanced interactive Python shell.')
     version = Unicode(release.version)
 
     aliases = Dict(base_aliases)
     flags = Dict(base_flags)
     classes = List([ProfileDir])
 
     # Track whether the config_file has changed,
     # because some logic happens only if we aren't using the default.
     config_file_specified = Set()
 
     config_file_name = Unicode()
     def _config_file_name_default(self):
         return self.name.replace('-','_') + u'_config.py'
     def _config_file_name_changed(self, name, old, new):
         if new != old:
             self.config_file_specified.add(new)
 
     # The directory that contains IPython's builtin profiles.
     builtin_profile_dir = Unicode(
         os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
     )
 
     config_file_paths = List(Unicode)
     def _config_file_paths_default(self):
         return [os.getcwdu()]
 
     extra_config_file = Unicode(config=True,
     help="""Path to an extra config file to load.
     
     If specified, load this config file in addition to any other IPython config.
     """)
     def _extra_config_file_changed(self, name, old, new):
         try:
             self.config_files.remove(old)
         except ValueError:
             pass
         self.config_file_specified.add(new)
         self.config_files.append(new)
 
     profile = Unicode(u'default', config=True,
         help="""The IPython profile to use."""
     )
 
     def _profile_changed(self, name, old, new):
         self.builtin_profile_dir = os.path.join(
                 get_ipython_package_dir(), u'config', u'profile', new
         )
 
     ipython_dir = Unicode(config=True,
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This options can also be specified through
         the environment variable IPYTHONDIR.
         """
     )
     def _ipython_dir_default(self):
         d = get_ipython_dir()
         self._ipython_dir_changed('ipython_dir', d, d)
         return d
     
     _in_init_profile_dir = False
     profile_dir = Instance(ProfileDir)
     def _profile_dir_default(self):
         # avoid recursion
         if self._in_init_profile_dir:
             return
         # profile_dir requested early, force initialization
         self.init_profile_dir()
         return self.profile_dir
 
     overwrite = Bool(False, config=True,
         help="""Whether to overwrite existing config files when copying""")
     auto_create = Bool(False, config=True,
         help="""Whether to create profile dir if it doesn't exist""")
 
     config_files = List(Unicode)
     def _config_files_default(self):
         return [self.config_file_name]
 
     copy_config_files = Bool(False, config=True,
         help="""Whether to install the default config files into the profile dir.
         If a new profile is being created, and IPython contains config files for that
         profile, then they will be staged into the new directory.  Otherwise,
         default config files will be automatically generated.
         """)
     
     verbose_crash = Bool(False, config=True,
         help="""Create a massive crash report when IPython encounters what may be an
         internal error.  The default is to append a short message to the
         usual traceback""")
 
     # The class to use as the crash handler.
     crash_handler_class = Type(crashhandler.CrashHandler)
 
     @catch_config_error
     def __init__(self, **kwargs):
         super(BaseIPythonApplication, self).__init__(**kwargs)
         # ensure current working directory exists
         try:
             directory = os.getcwdu()
         except:
             # raise exception
             self.log.error("Current working directory doesn't exist.")
             raise
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.excepthook
         def unset_crashhandler():
             sys.excepthook = sys.__excepthook__
         atexit.register(unset_crashhandler)
     
     def excepthook(self, etype, evalue, tb):
         """this is sys.excepthook after init_crashhandler
         
         set self.verbose_crash=True to use our full crashhandler, instead of
         a regular traceback with a short message (crash_handler_lite)
         """
         
         if self.verbose_crash:
             return self.crash_handler(etype, evalue, tb)
         else:
             return crashhandler.crash_handler_lite(etype, evalue, tb)
     
     def _ipython_dir_changed(self, name, old, new):
         if old in sys.path:
             sys.path.remove(old)
         sys.path.append(os.path.abspath(new))
         if not os.path.isdir(new):
             os.makedirs(new, mode=0o777)
         readme = os.path.join(new, 'README')
         readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
         if not os.path.exists(readme) and os.path.exists(readme_src):
             shutil.copy(readme_src, readme)
         for d in ('extensions', 'nbextensions'):
             path = os.path.join(new, d)
             if not os.path.exists(path):
                 try:
                     os.mkdir(path)
                 except OSError as e:
                     if e.errno != errno.EEXIST:
                         self.log.error("couldn't create path %s: %s", path, e)
         self.log.debug("IPYTHONDIR set to: %s" % new)
 
     def load_config_file(self, suppress_errors=True):
         """Load the config file.
 
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
         """
         self.log.debug("Searching path %s for config files", self.config_file_paths)
         base_config = 'ipython_config.py'
         self.log.debug("Attempting to load config file: %s" %
                        base_config)
         try:
             Application.load_config_file(
                 self,
                 base_config,
                 path=self.config_file_paths
             )
         except ConfigFileNotFound:
             # ignore errors loading parent
             self.log.debug("Config file %s not found", base_config)
             pass
         
         for config_file_name in self.config_files:
             if not config_file_name or config_file_name == base_config:
                 continue
             self.log.debug("Attempting to load config file: %s" %
                            self.config_file_name)
             try:
                 Application.load_config_file(
                     self,
                     config_file_name,
                     path=self.config_file_paths
                 )
             except ConfigFileNotFound:
                 # Only warn if the default config file was NOT being used.
                 if config_file_name in self.config_file_specified:
                     msg = self.log.warn
                 else:
                     msg = self.log.debug
                 msg("Config file not found, skipping: %s", config_file_name)
             except:
                 # For testing purposes.
                 if not suppress_errors:
                     raise
                 self.log.warn("Error loading config file: %s" %
                               self.config_file_name, exc_info=True)
 
     def init_profile_dir(self):
         """initialize the profile dir"""
         self._in_init_profile_dir = True
         if self.profile_dir is not None:
             # already ran
             return
-        try:
-            # location explicitly specified:
-            location = self.config.ProfileDir.location
-        except AttributeError:
+        if 'ProfileDir.location' not in self.config:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
             except ProfileDirError:
                 # not found, maybe create it (always create default profile)
                 if self.auto_create or self.profile == 'default':
                     try:
                         p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile: %r"%self.profile)
                         self.exit(1)
                     else:
                         self.log.info("Created profile dir: %r"%p.location)
                 else:
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%p.location)
         else:
+            location = self.config.ProfileDir.location
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)
             except ProfileDirError:
                 # not found, maybe create it
                 if self.auto_create:
                     try:
                         p = ProfileDir.create_profile_dir(location, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile directory: %r"%location)
                         self.exit(1)
                     else:
                         self.log.info("Creating new profile dir: %r"%location)
                 else:
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%location)
 
         self.profile_dir = p
         self.config_file_paths.append(p.location)
         self._in_init_profile_dir = False
 
     def init_config_files(self):
         """[optionally] copy default config files into profile dir."""
         # copy config files
         path = self.builtin_profile_dir
         if self.copy_config_files:
             src = self.profile
 
             cfg = self.config_file_name
             if path and os.path.exists(os.path.join(path, cfg)):
                 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
                         cfg, src, self.profile_dir.location, self.overwrite)
                 )
                 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
             else:
                 self.stage_default_config_file()
         else:
             # Still stage *bundled* config files, but not generated ones
             # This is necessary for `ipython profile=sympy` to load the profile
             # on the first go
             files = glob.glob(os.path.join(path, '*.py'))
             for fullpath in files:
                 cfg = os.path.basename(fullpath)
                 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                     # file was copied
                     self.log.warn("Staging bundled %s from %s into %r"%(
                             cfg, self.profile, self.profile_dir.location)
                     )
 
 
     def stage_default_config_file(self):
         """auto generate default config file, and stage it into the profile."""
         s = self.generate_config_file()
         fname = os.path.join(self.profile_dir.location, self.config_file_name)
         if self.overwrite or not os.path.exists(fname):
             self.log.warn("Generating default config file: %r"%(fname))
             with open(fname, 'w') as f:
                 f.write(s)
 
     @catch_config_error
     def initialize(self, argv=None):
         # don't hook up crash handler before parsing command-line
         self.parse_command_line(argv)
         self.init_crash_handler()
         if self.subapp is not None:
             # stop here if subapp is taking over
             return
         cl_config = self.config
         self.init_profile_dir()
         self.init_config_files()
         self.load_config_file()
         # enforce cl-opts override configfile opts:
         self.update_config(cl_config)
 

IPython/parallel/apps/ipcontrollerapp.py

@@ -1,547 +1,547 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import json
 import os
 import stat
 import sys
 
 from multiprocessing import Process
 from signal import signal, SIGINT, SIGABRT, SIGTERM
 
 import zmq
 from zmq.devices import ProcessMonitoredQueue
 from zmq.log.handlers import PUBHandler
 
 from IPython.core.profiledir import ProfileDir
 
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
     catch_config_error,
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import localhost, public_ips
 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
 
 from IPython.kernel.zmq.session import (
     Session, session_aliases, session_flags, default_secure
 )
 
 from IPython.parallel.controller.heartmonitor import HeartMonitor
 from IPython.parallel.controller.hub import HubFactory
 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
 from IPython.parallel.controller.dictdb import DictDB
 
 from IPython.parallel.util import split_url, disambiguate_url, set_hwm
 
 # conditional import of SQLiteDB / MongoDB backend class
 real_dbs = []
 
 try:
     from IPython.parallel.controller.sqlitedb import SQLiteDB
 except ImportError:
     pass
 else:
     real_dbs.append(SQLiteDB)
 
 try:
     from IPython.parallel.controller.mongodb import MongoDB
 except ImportError:
     pass
 else:
     real_dbs.append(MongoDB)
 
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your ipython directory and named as "profile_name". See the `profile`
 and `profile-dir` options for details.
 """
 
 _examples = """
 ipcontroller --ip=192.168.0.1 --port=1000  # listen on ip, port for engines
 ipcontroller --scheme=pure  # use the pure zeromq scheduler
 """
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 flags = {}
 flags.update(base_flags)
 flags.update({
     'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                     'Use threads instead of processes for the schedulers'),
     'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                     'use the SQLiteDB backend'),
     'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                     'use the MongoDB backend'),
     'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                     'use the in-memory DictDB backend'),
     'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
                     """use dummy DB backend, which doesn't store any information.
                     
                     This is the default as of IPython 0.13.
                     
                     To enable delayed or repeated retrieval of results from the Hub,
                     select one of the true db backends.
                     """),
     'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                     'reuse existing json connection files'),
     'restore' : ({'IPControllerApp' : {'restore_engines' : True, 'reuse_files' : True}},
                     'Attempt to restore engines from a JSON file.  '
                     'For use when resuming a crashed controller'),
 })
 
 flags.update(session_flags)
 
 aliases = dict(
     ssh = 'IPControllerApp.ssh_server',
     enginessh = 'IPControllerApp.engine_ssh_server',
     location = 'IPControllerApp.location',
 
     url = 'HubFactory.url',
     ip = 'HubFactory.ip',
     transport = 'HubFactory.transport',
     port = 'HubFactory.regport',
 
     ping = 'HeartMonitor.period',
 
     scheme = 'TaskScheduler.scheme_name',
     hwm = 'TaskScheduler.hwm',
 )
 aliases.update(base_aliases)
 aliases.update(session_aliases)
 
 class IPControllerApp(BaseParallelApplication):
 
     name = u'ipcontroller'
     description = _description
     examples = _examples
     classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs
     
     # change default to True
     auto_create = Bool(True, config=True,
         help="""Whether to create profile dir if it doesn't exist.""")
     
     reuse_files = Bool(False, config=True,
         help="""Whether to reuse existing json connection files.
         If False, connection files will be removed on a clean exit.
         """
     )
     restore_engines = Bool(False, config=True,
         help="""Reload engine state from JSON file
         """
     )
     ssh_server = Unicode(u'', config=True,
         help="""ssh url for clients to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     engine_ssh_server = Unicode(u'', config=True,
         help="""ssh url for engines to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     location = Unicode(u'', config=True,
         help="""The external IP or domain name of the Controller, used for disambiguating
         engine and client connections.""",
     )
     import_statements = List([], config=True,
         help="import statements to be run at startup.  Necessary in some environments"
     )
 
     use_threads = Bool(False, config=True,
         help='Use threads instead of processes for the schedulers',
     )
 
     engine_json_file = Unicode('ipcontroller-engine.json', config=True,
         help="JSON filename where engine connection info will be stored.")
     client_json_file = Unicode('ipcontroller-client.json', config=True,
         help="JSON filename where client connection info will be stored.")
     
     def _cluster_id_changed(self, name, old, new):
         super(IPControllerApp, self)._cluster_id_changed(name, old, new)
         self.engine_json_file = "%s-engine.json" % self.name
         self.client_json_file = "%s-client.json" % self.name
 
 
     # internal
     children = List()
     mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
 
     def _use_threads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
     
     write_connection_files = Bool(True,
         help="""Whether to write connection files to disk.
         True in all cases other than runs with `reuse_files=True` *after the first*
         """
     )
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     
 
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
         c = self.config
         url = cdict['registration']
         location = cdict['location']
         
         if not location:
             if public_ips():
                 location = public_ips()[-1]
             else:
                 self.log.warn("Could not identify this machine's IP, assuming %s."
                 " You may need to specify '--location=<external_ip_address>' to help"
                 " IPython decide when to connect via loopback." % localhost() )
                 location = localhost()
             cdict['location'] = location
         fname = os.path.join(self.profile_dir.security_dir, fname)
         self.log.info("writing connection info to %s", fname)
         with open(fname, 'w') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
         c = self.config
         self.log.debug("loading config from JSON")
         
         # load engine config
         
         fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             ecfg = json.loads(f.read())
         
         # json gives unicode, Session.key wants bytes
         c.Session.key = ecfg['key'].encode('ascii')
         
         xport,ip = ecfg['interface'].split('://')
         
         c.HubFactory.engine_ip = ip
         c.HubFactory.engine_transport = xport
         
         self.location = ecfg['location']
         if not self.engine_ssh_server:
             self.engine_ssh_server = ecfg['ssh']
         
         # load client config
         
         fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             ccfg = json.loads(f.read())
         
         for key in ('key', 'registration', 'pack', 'unpack', 'signature_scheme'):
             assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key
         
         xport,addr = ccfg['interface'].split('://')
         
         c.HubFactory.client_transport = xport
         c.HubFactory.client_ip = ip
         if not self.ssh_server:
             self.ssh_server = ccfg['ssh']
         
         # load port config:
         c.HubFactory.regport = ecfg['registration']
         c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong'])
         c.HubFactory.control = (ccfg['control'], ecfg['control'])
         c.HubFactory.mux = (ccfg['mux'], ecfg['mux'])
         c.HubFactory.task = (ccfg['task'], ecfg['task'])
         c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub'])
         c.HubFactory.notifier_port = ccfg['notification']
     
     def cleanup_connection_files(self):
         if self.reuse_files:
             self.log.debug("leaving JSON connection files for reuse")
             return
         self.log.debug("cleaning up JSON connection files")
         for f in (self.client_json_file, self.engine_json_file):
             f = os.path.join(self.profile_dir.security_dir, f)
             try:
                 os.remove(f)
             except Exception as e:
                 self.log.error("Failed to cleanup connection file: %s", e)
             else:
                 self.log.debug(u"removed %s", f)
     
     def load_secondary_config(self):
         """secondary config, loading from JSON and setting defaults"""
         if self.reuse_files:
             try:
                 self.load_config_from_json()
             except (AssertionError,IOError) as e:
                 self.log.error("Could not load config from JSON: %s" % e)
             else:
                 # successfully loaded config from JSON, and reuse=True
                 # no need to wite back the same file
                 self.write_connection_files = False
                 
         # switch Session.key default to secure
         default_secure(self.config)
         self.log.debug("Config changed")
         self.log.debug(repr(self.config))
         
     def init_hub(self):
         c = self.config
         
         self.do_import_statements()
         
         try:
             self.factory = HubFactory(config=c, log=self.log)
             # self.start_logging()
             self.factory.init_hub()
         except TraitError:
             raise
         except Exception:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
         
         if self.write_connection_files:
             # save to new json config files
             f = self.factory
             base = {
                 'key'  : f.session.key.decode('ascii'),
                 'location'  : self.location,
                 'pack'      : f.session.packer,
                 'unpack'    : f.session.unpacker,
                 'signature_scheme' : f.session.signature_scheme,
             }
             
             cdict = {'ssh' : self.ssh_server}
             cdict.update(f.client_info)
             cdict.update(base)
             self.save_connection_dict(self.client_json_file, cdict)
             
             edict = {'ssh' : self.engine_ssh_server}
             edict.update(f.engine_info)
             edict.update(base)
             self.save_connection_dict(self.engine_json_file, edict)
 
         fname = "engines%s.json" % self.cluster_id
         self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname)
         if self.restore_engines:
             self.factory.hub._load_engine_state()
 
     def init_schedulers(self):
         children = self.children
         mq = import_item(str(self.mq_class))
         
         f = self.factory
         ident = f.session.bsession
         # disambiguate url, in case of *
         monitor_url = disambiguate_url(f.monitor_url)
         # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
         q.bind_in(f.client_url('iopub'))
         q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub")
         q.bind_out(f.engine_url('iopub'))
         q.setsockopt_out(zmq.SUBSCRIBE, b'')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Multiplexer Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
         
         q.bind_in(f.client_url('mux'))
         q.setsockopt_in(zmq.IDENTITY, b'mux_in')
         q.bind_out(f.engine_url('mux'))
         q.setsockopt_out(zmq.IDENTITY, b'mux_out')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Control Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
         q.bind_in(f.client_url('control'))
         q.setsockopt_in(zmq.IDENTITY, b'control_in')
         q.bind_out(f.engine_url('control'))
         q.setsockopt_out(zmq.IDENTITY, b'control_out')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
-        try:
+        if 'TaskScheduler.scheme_name' in self.config:
             scheme = self.config.TaskScheduler.scheme_name
-        except AttributeError:
+        else:
             scheme = TaskScheduler.scheme_name.get_default_value()
         # Task Queue (in a Process)
         if scheme == 'pure':
             self.log.warn("task::using pure DEALER Task scheduler")
             q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
             # q.setsockopt_out(zmq.HWM, hub.hwm)
             q.bind_in(f.client_url('task'))
             q.setsockopt_in(zmq.IDENTITY, b'task_in')
             q.bind_out(f.engine_url('task'))
             q.setsockopt_out(zmq.IDENTITY, b'task_out')
             q.connect_mon(monitor_url)
             q.daemon=True
             children.append(q)
         elif scheme == 'none':
             self.log.warn("task::using no Task scheduler")
 
         else:
             self.log.info("task::using Python %s Task scheduler"%scheme)
             sargs = (f.client_url('task'), f.engine_url('task'),
                     monitor_url, disambiguate_url(f.client_url('notification')),
                     disambiguate_url(f.client_url('registration')),
             )
             kwargs = dict(logname='scheduler', loglevel=self.log_level,
                             log_url = self.log_url, config=dict(self.config))
             if 'Process' in self.mq_class:
                 # run the Python scheduler in a Process
                 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                 q.daemon=True
                 children.append(q)
             else:
                 # single-threaded Controller
                 kwargs['in_thread'] = True
                 launch_scheduler(*sargs, **kwargs)
         
         # set unlimited HWM for all relay devices
         if hasattr(zmq, 'SNDHWM'):
             q = children[0]
             q.setsockopt_in(zmq.RCVHWM, 0)
             q.setsockopt_out(zmq.SNDHWM, 0)
             
             for q in children[1:]:
                 if not hasattr(q, 'setsockopt_in'):
                     continue
                 q.setsockopt_in(zmq.SNDHWM, 0)
                 q.setsockopt_in(zmq.RCVHWM, 0)
                 q.setsockopt_out(zmq.SNDHWM, 0)
                 q.setsockopt_out(zmq.RCVHWM, 0)
                 q.setsockopt_mon(zmq.SNDHWM, 0)
             
 
     def terminate_children(self):
         child_procs = []
         for child in self.children:
             if isinstance(child, ProcessMonitoredQueue):
                 child_procs.append(child.launcher)
             elif isinstance(child, Process):
                 child_procs.append(child)
         if child_procs:
             self.log.critical("terminating children...")
             for child in child_procs:
                 try:
                     child.terminate()
                 except OSError:
                     # already dead
                     pass
     
     def handle_signal(self, sig, frame):
         self.log.critical("Received signal %i, shutting down", sig)
         self.terminate_children()
         self.loop.stop()
     
     def init_signal(self):
         for sig in (SIGINT, SIGABRT, SIGTERM):
             signal(sig, self.handle_signal)
 
     def do_import_statements(self):
         statements = self.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
                 exec s in globals(), locals()
             except:
                 self.log.msg("Error running statement: %s" % s)
 
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s"%self.log_url)
             context = zmq.Context.instance()
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             handler = PUBHandler(lsock)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
     
     @catch_config_error
     def initialize(self, argv=None):
         super(IPControllerApp, self).initialize(argv)
         self.forward_logging()
         self.load_secondary_config()
         self.init_hub()
         self.init_schedulers()
     
     def start(self):
         # Start the subprocesses:
         self.factory.start()
         # children must be started before signals are setup,
         # otherwise signal-handling will fire multiple times
         for child in self.children:
             child.start()
         self.init_signal()
 
         self.write_pid_file(overwrite=True)
 
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
         finally:
             self.cleanup_connection_files()
             
 
 def launch_new_instance(*args, **kwargs):
     """Create and run the IPython controller"""
     if sys.platform == 'win32':
         # make sure we don't get called from a multiprocessing subprocess
         # this can result in infinite Controllers being started on Windows
         # which doesn't have a proper fork, so multiprocessing is wonky
         
         # this only comes up when IPython has been installed using vanilla
         # setuptools, and *not* distribute.
         import multiprocessing
         p = multiprocessing.current_process()
         # the main process has name 'MainProcess'
         # subprocesses will have names like 'Process-1'
         if p.name != 'MainProcess':
             # we are a subprocess, don't start another Controller!
             return
     return IPControllerApp.launch_instance(*args, **kwargs)
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/parallel/apps/ipengineapp.py

@@ -1,393 +1,384 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython engine application
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import json
 import os
 import sys
 import time
 
 import zmq
 from zmq.eventloop import ioloop
 
 from IPython.core.profiledir import ProfileDir
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
     catch_config_error,
 )
 from IPython.kernel.zmq.log import EnginePUBHandler
 from IPython.kernel.zmq.ipkernel import Kernel
 from IPython.kernel.zmq.kernelapp import IPKernelApp
 from IPython.kernel.zmq.session import (
     Session, session_aliases, session_flags
 )
 from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell
 
 from IPython.config.configurable import Configurable
 
 from IPython.parallel.engine.engine import EngineFactory
 from IPython.parallel.util import disambiguate_ip_address
 
 from IPython.utils.importstring import import_item
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 _description = """Start an IPython engine for parallel computing.
 
 IPython engines run in parallel and perform computations on behalf of a client
 and controller. A controller needs to be started before the engines. The
 engine can be configured using command line options or using a cluster
 directory. Cluster directories contain config, log and security files and are
 usually located in your ipython directory and named as "profile_name".
 See the `profile` and `profile-dir` options for details.
 """
 
 _examples = """
 ipengine --ip=192.168.0.1 --port=1000     # connect to hub at ip and port
 ipengine --log-to-file --log-level=DEBUG  # log to a file with DEBUG verbosity
 """
 
 #-----------------------------------------------------------------------------
 # MPI configuration
 #-----------------------------------------------------------------------------
 
 mpi4py_init = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """
 
 
 pytrilinos_init = """from PyTrilinos import Epetra
 class SimpleStruct:
 pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """
 
 class MPI(Configurable):
     """Configurable for MPI initialization"""
     use = Unicode('', config=True,
         help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
         )
 
     def _use_changed(self, name, old, new):
         # load default init script if it's not set
         if not self.init_script:
             self.init_script = self.default_inits.get(new, '')
 
     init_script = Unicode('', config=True,
         help="Initialization code for MPI")
 
     default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
         config=True)
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 aliases = dict(
     file = 'IPEngineApp.url_file',
     c = 'IPEngineApp.startup_command',
     s = 'IPEngineApp.startup_script',
 
     url = 'EngineFactory.url',
     ssh = 'EngineFactory.sshserver',
     sshkey = 'EngineFactory.sshkey',
     ip = 'EngineFactory.ip',
     transport = 'EngineFactory.transport',
     port = 'EngineFactory.regport',
     location = 'EngineFactory.location',
 
     timeout = 'EngineFactory.timeout',
 
     mpi = 'MPI.use',
 
 )
 aliases.update(base_aliases)
 aliases.update(session_aliases)
 flags = {}
 flags.update(base_flags)
 flags.update(session_flags)
 
 class IPEngineApp(BaseParallelApplication):
 
     name = 'ipengine'
     description = _description
     examples = _examples
     classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI])
 
     startup_script = Unicode(u'', config=True,
         help='specify a script to be run at startup')
     startup_command = Unicode('', config=True,
             help='specify a command to be run at startup')
 
     url_file = Unicode(u'', config=True,
         help="""The full location of the file containing the connection information for
         the controller. If this is not given, the file must be in the
         security directory of the cluster directory.  This location is
         resolved using the `profile` or `profile_dir` options.""",
         )
     wait_for_url_file = Float(5, config=True,
         help="""The maximum number of seconds to wait for url_file to exist.
         This is useful for batch-systems and shared-filesystems where the
         controller and engine are started at the same time and it
         may take a moment for the controller to write the connector files.""")
 
     url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
 
     def _cluster_id_changed(self, name, old, new):
         if new:
             base = 'ipcontroller-%s' % new
         else:
             base = 'ipcontroller'
         self.url_file_name = "%s-engine.json" % base
 
     log_url = Unicode('', config=True,
         help="""The URL for the iploggerapp instance, for forwarding
         logging to a central location.""")
     
     # an IPKernelApp instance, used to setup listening for shell frontends
     kernel_app = Instance(IPKernelApp)
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     
     @property
     def kernel(self):
         """allow access to the Kernel object, so I look like IPKernelApp"""
         return self.engine.kernel
 
     def find_url_file(self):
         """Set the url file.
 
         Here we don't try to actually see if it exists for is valid as that
         is hadled by the connection logic.
         """
         config = self.config
         # Find the actual controller key file
         if not self.url_file:
             self.url_file = os.path.join(
                 self.profile_dir.security_dir,
                 self.url_file_name
             )
     
     def load_connector_file(self):
         """load config from a JSON connector file,
         at a *lower* priority than command-line/config files.
         """
         
         self.log.info("Loading url_file %r", self.url_file)
         config = self.config
         
         with open(self.url_file) as f:
             d = json.loads(f.read())
         
         # allow hand-override of location for disambiguation
         # and ssh-server
-        try:
-            config.EngineFactory.location
-        except AttributeError:
+        if 'EngineFactory.location' not in config:
             config.EngineFactory.location = d['location']
-        
-        try:
-            config.EngineFactory.sshserver
-        except AttributeError:
+        if 'EngineFactory.sshserver' not in config:
             config.EngineFactory.sshserver = d.get('ssh')
         
         location = config.EngineFactory.location
         
         proto, ip = d['interface'].split('://')
         ip = disambiguate_ip_address(ip, location)
         d['interface'] = '%s://%s' % (proto, ip)
         
         # DO NOT allow override of basic URLs, serialization, or key
         # JSON file takes top priority there
         config.Session.key = cast_bytes(d['key'])
         config.Session.signature_scheme = d['signature_scheme']
         
         config.EngineFactory.url = d['interface'] + ':%i' % d['registration']
         
         config.Session.packer = d['pack']
         config.Session.unpacker = d['unpack']
         
         self.log.debug("Config changed:")
         self.log.debug("%r", config)
         self.connection_info = d
     
     def bind_kernel(self, **kwargs):
         """Promote engine to listening kernel, accessible to frontends."""
         if self.kernel_app is not None:
             return
         
         self.log.info("Opening ports for direct connections as an IPython kernel")
         
         kernel = self.kernel
         
         kwargs.setdefault('config', self.config)
         kwargs.setdefault('log', self.log)
         kwargs.setdefault('profile_dir', self.profile_dir)
         kwargs.setdefault('session', self.engine.session)
         
         app = self.kernel_app = IPKernelApp(**kwargs)
         
         # allow IPKernelApp.instance():
         IPKernelApp._instance = app
         
         app.init_connection_file()
         # relevant contents of init_sockets:
         
         app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port)
         app.log.debug("shell ROUTER Channel on port: %i", app.shell_port)
         
         app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)
         app.log.debug("iopub PUB Channel on port: %i", app.iopub_port)
         
         kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)
         app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)
         app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port)
         
         # start the heartbeat, and log connection info:
         
         app.init_heartbeat()
         
         app.log_connection_info()
         app.write_connection_file()
         
     
     def init_engine(self):
         # This is the working dir by now.
         sys.path.insert(0, '')
         config = self.config
         # print config
         self.find_url_file()
         
         # was the url manually specified?
         keys = set(self.config.EngineFactory.keys())
         keys = keys.union(set(self.config.RegistrationFactory.keys()))
         
         if keys.intersection(set(['ip', 'url', 'port'])):
             # Connection info was specified, don't wait for the file
             url_specified = True
             self.wait_for_url_file = 0
         else:
             url_specified = False
 
         if self.wait_for_url_file and not os.path.exists(self.url_file):
             self.log.warn("url_file %r not found", self.url_file)
             self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
             tic = time.time()
             while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
                 # wait for url_file to exist, or until time limit
                 time.sleep(0.1)
             
         if os.path.exists(self.url_file):
             self.load_connector_file()
         elif not url_specified:
             self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
             self.exit(1)
         
+        exec_lines = []
+        for app in ('IPKernelApp', 'InteractiveShellApp'):
+            if '%s.exec_lines' in config:
+                exec_lines = config.IPKernelApp.exec_lines = config[app].exec_lines
+                break
         
-        try:
-            exec_lines = config.IPKernelApp.exec_lines
-        except AttributeError:
-            try:
-                exec_lines = config.InteractiveShellApp.exec_lines
-            except AttributeError:
-                exec_lines = config.IPKernelApp.exec_lines = []
-        try:
-            exec_files = config.IPKernelApp.exec_files
-        except AttributeError:
-            try:
-                exec_files = config.InteractiveShellApp.exec_files
-            except AttributeError:
-                exec_files = config.IPKernelApp.exec_files = []
+        exec_files = []
+        for app in ('IPKernelApp', 'InteractiveShellApp'):
+            if '%s.exec_files' in config:
+                exec_files = config.IPKernelApp.exec_files = config[app].exec_files
+                break
         
         if self.startup_script:
             exec_files.append(self.startup_script)
         if self.startup_command:
             exec_lines.append(self.startup_command)
 
         # Create the underlying shell class and Engine
         # shell_class = import_item(self.master_config.Global.shell_class)
         # print self.config
         try:
             self.engine = EngineFactory(config=config, log=self.log,
                             connection_info=self.connection_info,
                         )
         except:
             self.log.error("Couldn't start the Engine", exc_info=True)
             self.exit(1)
     
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s", self.log_url)
             context = self.engine.context
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             handler = EnginePUBHandler(self.engine, lsock)
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
     
     def init_mpi(self):
         global mpi
         self.mpi = MPI(parent=self)
 
         mpi_import_statement = self.mpi.init_script
         if mpi_import_statement:
             try:
                 self.log.info("Initializing MPI:")
                 self.log.info(mpi_import_statement)
                 exec mpi_import_statement in globals()
             except:
                 mpi = None
         else:
             mpi = None
 
     @catch_config_error
     def initialize(self, argv=None):
         super(IPEngineApp, self).initialize(argv)
         self.init_mpi()
         self.init_engine()
         self.forward_logging()
     
     def start(self):
         self.engine.start()
         try:
             self.engine.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Engine Interrupted, shutting down...\n")
 
 
 launch_new_instance = IPEngineApp.launch_instance
 
 
 if __name__ == '__main__':
     launch_new_instance()
 

IPython/parallel/controller/hub.py

@@ -1,1422 +1,1421 b''
 """The IPython Controller Hub with 0MQ
 This is the master object that handles connections from engines and clients,
 and monitors traffic through the various queues.
 
 Authors:
 
 * Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import json
 import os
 import sys
 import time
 from datetime import datetime
 
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 # internal:
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import localhost
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.traitlets import (
         HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName
         )
 
 from IPython.parallel import error, util
 from IPython.parallel.factory import RegistrationFactory
 
 from IPython.kernel.zmq.session import SessionFactory
 
 from .heartmonitor import HeartMonitor
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 def _passer(*args, **kwargs):
     return
 
 def _printer(*args, **kwargs):
     print (args)
     print (kwargs)
 
 def empty_record():
     """Return an empty dict with all record keys."""
     return {
         'msg_id' : None,
         'header' : None,
         'metadata' : None,
         'content': None,
         'buffers': None,
         'submitted': None,
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'received': None,
         'result_header' : None,
         'result_metadata' : None,
         'result_content' : None,
         'result_buffers' : None,
         'queue' : None,
         'pyin' : None,
         'pyout': None,
         'pyerr': None,
         'stdout': '',
         'stderr': '',
     }
 
 def init_record(msg):
     """Initialize a TaskRecord based on a request."""
     header = msg['header']
     return {
         'msg_id' : header['msg_id'],
         'header' : header,
         'content': msg['content'],
         'metadata': msg['metadata'],
         'buffers': msg['buffers'],
         'submitted': header['date'],
         'client_uuid' : None,
         'engine_uuid' : None,
         'started': None,
         'completed': None,
         'resubmitted': None,
         'received': None,
         'result_header' : None,
         'result_metadata': None,
         'result_content' : None,
         'result_buffers' : None,
         'queue' : None,
         'pyin' : None,
         'pyout': None,
         'pyerr': None,
         'stdout': '',
         'stderr': '',
     }
 
 
 class EngineConnector(HasTraits):
     """A simple object for accessing the various zmq connections of an object.
     Attributes are:
     id (int): engine ID
     uuid (unicode): engine UUID
     pending: set of msg_ids
     stallback: DelayedCallback for stalled registration
     """
     
     id = Integer(0)
     uuid = Unicode()
     pending = Set()
     stallback = Instance(ioloop.DelayedCallback)
 
 
 _db_shortcuts = {
     'sqlitedb' : 'IPython.parallel.controller.sqlitedb.SQLiteDB',
     'mongodb'  : 'IPython.parallel.controller.mongodb.MongoDB',
     'dictdb'   : 'IPython.parallel.controller.dictdb.DictDB',
     'nodb'     : 'IPython.parallel.controller.dictdb.NoDB',
 }
 
 class HubFactory(RegistrationFactory):
     """The Configurable for setting up a Hub."""
 
     # port-pairs for monitoredqueues:
     hb = Tuple(Integer,Integer,config=True,
         help="""PUB/ROUTER Port pair for Engine heartbeats""")
     def _hb_default(self):
         return tuple(util.select_random_ports(2))
 
     mux = Tuple(Integer,Integer,config=True,
         help="""Client/Engine Port pair for MUX queue""")
 
     def _mux_default(self):
         return tuple(util.select_random_ports(2))
 
     task = Tuple(Integer,Integer,config=True,
         help="""Client/Engine Port pair for Task queue""")
     def _task_default(self):
         return tuple(util.select_random_ports(2))
 
     control = Tuple(Integer,Integer,config=True,
         help="""Client/Engine Port pair for Control queue""")
 
     def _control_default(self):
         return tuple(util.select_random_ports(2))
 
     iopub = Tuple(Integer,Integer,config=True,
         help="""Client/Engine Port pair for IOPub relay""")
 
     def _iopub_default(self):
         return tuple(util.select_random_ports(2))
 
     # single ports:
     mon_port = Integer(config=True,
         help="""Monitor (SUB) port for queue traffic""")
 
     def _mon_port_default(self):
         return util.select_random_ports(1)[0]
 
     notifier_port = Integer(config=True,
         help="""PUB port for sending engine status notifications""")
 
     def _notifier_port_default(self):
         return util.select_random_ports(1)[0]
 
     engine_ip = Unicode(config=True,
         help="IP on which to listen for engine connections. [default: loopback]")
     def _engine_ip_default(self):
         return localhost()
     engine_transport = Unicode('tcp', config=True,
         help="0MQ transport for engine connections. [default: tcp]")
 
     client_ip = Unicode(config=True,
         help="IP on which to listen for client connections. [default: loopback]")
     client_transport = Unicode('tcp', config=True,
         help="0MQ transport for client connections. [default : tcp]")
 
     monitor_ip = Unicode(config=True,
         help="IP on which to listen for monitor messages. [default: loopback]")
     monitor_transport = Unicode('tcp', config=True,
         help="0MQ transport for monitor messages. [default : tcp]")
     
     _client_ip_default = _monitor_ip_default = _engine_ip_default
     
 
     monitor_url = Unicode('')
 
     db_class = DottedObjectName('NoDB',
         config=True, help="""The class to use for the DB backend
         
         Options include:
         
         SQLiteDB: SQLite
         MongoDB : use MongoDB
         DictDB  : in-memory storage (fastest, but be mindful of memory growth of the Hub)
         NoDB    : disable database altogether (default)
         
         """)
 
     # not configurable
     db = Instance('IPython.parallel.controller.dictdb.BaseDB')
     heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
 
     def _ip_changed(self, name, old, new):
         self.engine_ip = new
         self.client_ip = new
         self.monitor_ip = new
         self._update_monitor_url()
 
     def _update_monitor_url(self):
         self.monitor_url = "%s://%s:%i" % (self.monitor_transport, self.monitor_ip, self.mon_port)
 
     def _transport_changed(self, name, old, new):
         self.engine_transport = new
         self.client_transport = new
         self.monitor_transport = new
         self._update_monitor_url()
 
     def __init__(self, **kwargs):
         super(HubFactory, self).__init__(**kwargs)
         self._update_monitor_url()
 
 
     def construct(self):
         self.init_hub()
 
     def start(self):
         self.heartmonitor.start()
         self.log.info("Heartmonitor started")
 
     def client_url(self, channel):
         """return full zmq url for a named client channel"""
         return "%s://%s:%i" % (self.client_transport, self.client_ip, self.client_info[channel])
     
     def engine_url(self, channel):
         """return full zmq url for a named engine channel"""
         return "%s://%s:%i" % (self.engine_transport, self.engine_ip, self.engine_info[channel])
     
     def init_hub(self):
         """construct Hub object"""
 
         ctx = self.context
         loop = self.loop
-
-        try:
+        if 'TaskScheduler.scheme_name' in self.config:
             scheme = self.config.TaskScheduler.scheme_name
-        except AttributeError:
+        else:
             from .scheduler import TaskScheduler
             scheme = TaskScheduler.scheme_name.get_default_value()
         
         # build connection dicts
         engine = self.engine_info = {
             'interface'     : "%s://%s" % (self.engine_transport, self.engine_ip),
             'registration'  : self.regport,
             'control'       : self.control[1],
             'mux'           : self.mux[1],
             'hb_ping'       : self.hb[0],
             'hb_pong'       : self.hb[1],
             'task'          : self.task[1],
             'iopub'         : self.iopub[1],
             }
 
         client = self.client_info = {
             'interface'     : "%s://%s" % (self.client_transport, self.client_ip),
             'registration'  : self.regport,
             'control'       : self.control[0],
             'mux'           : self.mux[0],
             'task'          : self.task[0],
             'task_scheme'   : scheme,
             'iopub'         : self.iopub[0],
             'notification'  : self.notifier_port,
             }
         
         self.log.debug("Hub engine addrs: %s", self.engine_info)
         self.log.debug("Hub client addrs: %s", self.client_info)
         
         # Registrar socket
         q = ZMQStream(ctx.socket(zmq.ROUTER), loop)
         util.set_hwm(q, 0)
         q.bind(self.client_url('registration'))
         self.log.info("Hub listening on %s for registration.", self.client_url('registration'))
         if self.client_ip != self.engine_ip:
             q.bind(self.engine_url('registration'))
             self.log.info("Hub listening on %s for registration.", self.engine_url('registration'))
 
         ### Engine connections ###
 
         # heartbeat
         hpub = ctx.socket(zmq.PUB)
         hpub.bind(self.engine_url('hb_ping'))
         hrep = ctx.socket(zmq.ROUTER)
         util.set_hwm(hrep, 0)
         hrep.bind(self.engine_url('hb_pong'))
         self.heartmonitor = HeartMonitor(loop=loop, parent=self, log=self.log,
                                 pingstream=ZMQStream(hpub,loop),
                                 pongstream=ZMQStream(hrep,loop)
                             )
 
         ### Client connections ###
         
         # Notifier socket
         n = ZMQStream(ctx.socket(zmq.PUB), loop)
         n.bind(self.client_url('notification'))
 
         ### build and launch the queues ###
 
         # monitor socket
         sub = ctx.socket(zmq.SUB)
         sub.setsockopt(zmq.SUBSCRIBE, b"")
         sub.bind(self.monitor_url)
         sub.bind('inproc://monitor')
         sub = ZMQStream(sub, loop)
 
         # connect the db
         db_class = _db_shortcuts.get(self.db_class.lower(), self.db_class)
         self.log.info('Hub using DB backend: %r', (db_class.split('.')[-1]))
         self.db = import_item(str(db_class))(session=self.session.session,
                                             parent=self, log=self.log)
         time.sleep(.25)
 
         # resubmit stream
         r = ZMQStream(ctx.socket(zmq.DEALER), loop)
         url = util.disambiguate_url(self.client_url('task'))
         r.connect(url)
 
         self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                 query=q, notifier=n, resubmit=r, db=self.db,
                 engine_info=self.engine_info, client_info=self.client_info,
                 log=self.log)
 
 
 class Hub(SessionFactory):
     """The IPython Controller Hub with 0MQ connections
 
     Parameters
     ==========
     loop: zmq IOLoop instance
     session: Session object
     <removed> context: zmq context for creating new connections (?)
     queue: ZMQStream for monitoring the command queue (SUB)
     query: ZMQStream for engine registration and client queries requests (ROUTER)
     heartbeat: HeartMonitor object checking the pulse of the engines
     notifier: ZMQStream for broadcasting engine registration changes (PUB)
     db: connection to db for out of memory logging of commands
                 NotImplemented
     engine_info: dict of zmq connection information for engines to connect
                 to the queues.
     client_info: dict of zmq connection information for engines to connect
                 to the queues.
     """
     
     engine_state_file = Unicode()
     
     # internal data structures:
     ids=Set() # engine IDs
     keytable=Dict()
     by_ident=Dict()
     engines=Dict()
     clients=Dict()
     hearts=Dict()
     pending=Set()
     queues=Dict()  # pending msg_ids keyed by engine_id
     tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id
     completed=Dict() # completed msg_ids keyed by engine_id
     all_completed=Set() # completed msg_ids keyed by engine_id
     dead_engines=Set() # completed msg_ids keyed by engine_id
     unassigned=Set() # set of task msg_ds not yet assigned a destination
     incoming_registrations=Dict()
     registration_timeout=Integer()
     _idcounter=Integer(0)
 
     # objects from constructor:
     query=Instance(ZMQStream)
     monitor=Instance(ZMQStream)
     notifier=Instance(ZMQStream)
     resubmit=Instance(ZMQStream)
     heartmonitor=Instance(HeartMonitor)
     db=Instance(object)
     client_info=Dict()
     engine_info=Dict()
 
 
     def __init__(self, **kwargs):
         """
         # universal:
         loop: IOLoop for creating future connections
         session: streamsession for sending serialized data
         # engine:
         queue: ZMQStream for monitoring queue messages
         query: ZMQStream for engine+client registration and client requests
         heartbeat: HeartMonitor object for tracking engines
         # extra:
         db: ZMQStream for db connection (NotImplemented)
         engine_info: zmq address/protocol dict for engine connections
         client_info: zmq address/protocol dict for client connections
         """
 
         super(Hub, self).__init__(**kwargs)
         self.registration_timeout = max(10000, 5*self.heartmonitor.period)
 
         # register our callbacks
         self.query.on_recv(self.dispatch_query)
         self.monitor.on_recv(self.dispatch_monitor_traffic)
 
         self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure)
         self.heartmonitor.add_new_heart_handler(self.handle_new_heart)
 
         self.monitor_handlers = {b'in' : self.save_queue_request,
                                 b'out': self.save_queue_result,
                                 b'intask': self.save_task_request,
                                 b'outtask': self.save_task_result,
                                 b'tracktask': self.save_task_destination,
                                 b'incontrol': _passer,
                                 b'outcontrol': _passer,
                                 b'iopub': self.save_iopub_message,
         }
 
         self.query_handlers = {'queue_request': self.queue_status,
                                 'result_request': self.get_results,
                                 'history_request': self.get_history,
                                 'db_request': self.db_query,
                                 'purge_request': self.purge_results,
                                 'load_request': self.check_load,
                                 'resubmit_request': self.resubmit_task,
                                 'shutdown_request': self.shutdown_request,
                                 'registration_request' : self.register_engine,
                                 'unregistration_request' : self.unregister_engine,
                                 'connection_request': self.connection_request,
         }
 
         # ignore resubmit replies
         self.resubmit.on_recv(lambda msg: None, copy=False)
 
         self.log.info("hub::created hub")
     
     @property
     def _next_id(self):
         """gemerate a new ID.
 
         No longer reuse old ids, just count from 0."""
         newid = self._idcounter
         self._idcounter += 1
         return newid
         # newid = 0
         # incoming = [id[0] for id in self.incoming_registrations.itervalues()]
         # # print newid, self.ids, self.incoming_registrations
         # while newid in self.ids or newid in incoming:
         #     newid += 1
         # return newid
     
     #-----------------------------------------------------------------------------
     # message validation
     #-----------------------------------------------------------------------------
 
     def _validate_targets(self, targets):
         """turn any valid targets argument into a list of integer ids"""
         if targets is None:
             # default to all
             return self.ids
 
         if isinstance(targets, (int,str,unicode)):
             # only one target specified
             targets = [targets]
         _targets = []
         for t in targets:
             # map raw identities to ids
             if isinstance(t, (str,unicode)):
                 t = self.by_ident.get(cast_bytes(t), t)
             _targets.append(t)
         targets = _targets
         bad_targets = [ t for t in targets if t not in self.ids ]
         if bad_targets:
             raise IndexError("No Such Engine: %r" % bad_targets)
         if not targets:
             raise IndexError("No Engines Registered")
         return targets
 
     #-----------------------------------------------------------------------------
     # dispatch methods (1 per stream)
     #-----------------------------------------------------------------------------
 
 
     @util.log_errors
     def dispatch_monitor_traffic(self, msg):
         """all ME and Task queue messages come through here, as well as
         IOPub traffic."""
         self.log.debug("monitor traffic: %r", msg[0])
         switch = msg[0]
         try:
             idents, msg = self.session.feed_identities(msg[1:])
         except ValueError:
             idents=[]
         if not idents:
             self.log.error("Monitor message without topic: %r", msg)
             return
         handler = self.monitor_handlers.get(switch, None)
         if handler is not None:
             handler(idents, msg)
         else:
             self.log.error("Unrecognized monitor topic: %r", switch)
 
 
     @util.log_errors
     def dispatch_query(self, msg):
         """Route registration requests and queries from clients."""
         try:
             idents, msg = self.session.feed_identities(msg)
         except ValueError:
             idents = []
         if not idents:
             self.log.error("Bad Query Message: %r", msg)
             return
         client_id = idents[0]
         try:
             msg = self.session.unserialize(msg, content=True)
         except Exception:
             content = error.wrap_exception()
             self.log.error("Bad Query Message: %r", msg, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id,
                     content=content)
             return
         # print client_id, header, parent, content
         #switch on message type:
         msg_type = msg['header']['msg_type']
         self.log.info("client::client %r requested %r", client_id, msg_type)
         handler = self.query_handlers.get(msg_type, None)
         try:
             assert handler is not None, "Bad Message Type: %r" % msg_type
         except:
             content = error.wrap_exception()
             self.log.error("Bad Message Type: %r", msg_type, exc_info=True)
             self.session.send(self.query, "hub_error", ident=client_id,
                     content=content)
             return
 
         else:
             handler(idents, msg)
 
     def dispatch_db(self, msg):
         """"""
         raise NotImplementedError
 
     #---------------------------------------------------------------------------
     # handler methods (1 per event)
     #---------------------------------------------------------------------------
 
     #----------------------- Heartbeat --------------------------------------
 
     def handle_new_heart(self, heart):
         """handler to attach to heartbeater.
         Called when a new heart starts to beat.
         Triggers completion of registration."""
         self.log.debug("heartbeat::handle_new_heart(%r)", heart)
         if heart not in self.incoming_registrations:
             self.log.info("heartbeat::ignoring new heart: %r", heart)
         else:
             self.finish_registration(heart)
 
 
     def handle_heart_failure(self, heart):
         """handler to attach to heartbeater.
         called when a previously registered heart fails to respond to beat request.
         triggers unregistration"""
         self.log.debug("heartbeat::handle_heart_failure(%r)", heart)
         eid = self.hearts.get(heart, None)
         uuid = self.engines[eid].uuid
         if eid is None or self.keytable[eid] in self.dead_engines:
             self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart)
         else:
             self.unregister_engine(heart, dict(content=dict(id=eid, queue=uuid)))
 
     #----------------------- MUX Queue Traffic ------------------------------
 
     def save_queue_request(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %r", idents)
             return
         queue_id, client_id = idents[:2]
         try:
             msg = self.session.unserialize(msg)
         except Exception:
             self.log.error("queue::client %r sent invalid message to %r: %r", client_id, queue_id, msg, exc_info=True)
             return
 
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::target %r not registered", queue_id)
             self.log.debug("queue::    valid are: %r", self.by_ident.keys())
             return
         record = init_record(msg)
         msg_id = record['msg_id']
         self.log.info("queue::client %r submitted request %r to %s", client_id, msg_id, eid)
         # Unicode in records
         record['engine_uuid'] = queue_id.decode('ascii')
         record['client_uuid'] = msg['header']['session']
         record['queue'] = 'mux'
 
         try:
             # it's posible iopub arrived first:
             existing = self.db.get_record(msg_id)
             for key,evalue in existing.iteritems():
                 rvalue = record.get(key, None)
                 if evalue and rvalue and evalue != rvalue:
                     self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue)
                 elif evalue and not rvalue:
                     record[key] = evalue
             try:
                 self.db.update_record(msg_id, record)
             except Exception:
                 self.log.error("DB Error updating record %r", msg_id, exc_info=True)
         except KeyError:
             try:
                 self.db.add_record(msg_id, record)
             except Exception:
                 self.log.error("DB Error adding record %r", msg_id, exc_info=True)
 
 
         self.pending.add(msg_id)
         self.queues[eid].append(msg_id)
 
     def save_queue_result(self, idents, msg):
         if len(idents) < 2:
             self.log.error("invalid identity prefix: %r", idents)
             return
 
         client_id, queue_id = idents[:2]
         try:
             msg = self.session.unserialize(msg)
         except Exception:
             self.log.error("queue::engine %r sent invalid message to %r: %r",
                     queue_id, client_id, msg, exc_info=True)
             return
 
         eid = self.by_ident.get(queue_id, None)
         if eid is None:
             self.log.error("queue::unknown engine %r is sending a reply: ", queue_id)
             return
 
         parent = msg['parent_header']
         if not parent:
             return
         msg_id = parent['msg_id']
         if msg_id in self.pending:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             self.queues[eid].remove(msg_id)
             self.completed[eid].append(msg_id)
             self.log.info("queue::request %r completed on %s", msg_id, eid)
         elif msg_id not in self.all_completed:
             # it could be a result from a dead engine that died before delivering the
             # result
             self.log.warn("queue:: unknown msg finished %r", msg_id)
             return
         # update record anyway, because the unregistration could have been premature
         rheader = msg['header']
         md = msg['metadata']
         completed = rheader['date']
         started = md.get('started', None)
         result = {
             'result_header' : rheader,
             'result_metadata': md,
             'result_content': msg['content'],
             'received': datetime.now(),
             'started' : started,
             'completed' : completed
         }
 
         result['result_buffers'] = msg['buffers']
         try:
             self.db.update_record(msg_id, result)
         except Exception:
             self.log.error("DB Error updating record %r", msg_id, exc_info=True)
 
 
     #--------------------- Task Queue Traffic ------------------------------
 
     def save_task_request(self, idents, msg):
         """Save the submission of a task."""
         client_id = idents[0]
 
         try:
             msg = self.session.unserialize(msg)
         except Exception:
             self.log.error("task::client %r sent invalid task message: %r",
                     client_id, msg, exc_info=True)
             return
         record = init_record(msg)
 
         record['client_uuid'] = msg['header']['session']
         record['queue'] = 'task'
         header = msg['header']
         msg_id = header['msg_id']
         self.pending.add(msg_id)
         self.unassigned.add(msg_id)
         try:
             # it's posible iopub arrived first:
             existing = self.db.get_record(msg_id)
             if existing['resubmitted']:
                 for key in ('submitted', 'client_uuid', 'buffers'):
                     # don't clobber these keys on resubmit
                     # submitted and client_uuid should be different
                     # and buffers might be big, and shouldn't have changed
                     record.pop(key)
                     # still check content,header which should not change
                     # but are not expensive to compare as buffers
 
             for key,evalue in existing.iteritems():
                 if key.endswith('buffers'):
                     # don't compare buffers
                     continue
                 rvalue = record.get(key, None)
                 if evalue and rvalue and evalue != rvalue:
                     self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue)
                 elif evalue and not rvalue:
                     record[key] = evalue
             try:
                 self.db.update_record(msg_id, record)
             except Exception:
                 self.log.error("DB Error updating record %r", msg_id, exc_info=True)
         except KeyError:
             try:
                 self.db.add_record(msg_id, record)
             except Exception:
                 self.log.error("DB Error adding record %r", msg_id, exc_info=True)
         except Exception:
             self.log.error("DB Error saving task request %r", msg_id, exc_info=True)
 
     def save_task_result(self, idents, msg):
         """save the result of a completed task."""
         client_id = idents[0]
         try:
             msg = self.session.unserialize(msg)
         except Exception:
             self.log.error("task::invalid task result message send to %r: %r",
                     client_id, msg, exc_info=True)
             return
 
         parent = msg['parent_header']
         if not parent:
             # print msg
             self.log.warn("Task %r had no parent!", msg)
             return
         msg_id = parent['msg_id']
         if msg_id in self.unassigned:
             self.unassigned.remove(msg_id)
 
         header = msg['header']
         md = msg['metadata']
         engine_uuid = md.get('engine', u'')
         eid = self.by_ident.get(cast_bytes(engine_uuid), None)
         
         status = md.get('status', None)
 
         if msg_id in self.pending:
             self.log.info("task::task %r finished on %s", msg_id, eid)
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             if eid is not None:
                 if status != 'aborted':
                     self.completed[eid].append(msg_id)
                 if msg_id in self.tasks[eid]:
                     self.tasks[eid].remove(msg_id)
             completed = header['date']
             started = md.get('started', None)
             result = {
                 'result_header' : header,
                 'result_metadata': msg['metadata'],
                 'result_content': msg['content'],
                 'started' : started,
                 'completed' : completed,
                 'received' : datetime.now(),
                 'engine_uuid': engine_uuid,
             }
 
             result['result_buffers'] = msg['buffers']
             try:
                 self.db.update_record(msg_id, result)
             except Exception:
                 self.log.error("DB Error saving task request %r", msg_id, exc_info=True)
 
         else:
             self.log.debug("task::unknown task %r finished", msg_id)
 
     def save_task_destination(self, idents, msg):
         try:
             msg = self.session.unserialize(msg, content=True)
         except Exception:
             self.log.error("task::invalid task tracking message", exc_info=True)
             return
         content = msg['content']
         # print (content)
         msg_id = content['msg_id']
         engine_uuid = content['engine_id']
         eid = self.by_ident[cast_bytes(engine_uuid)]
 
         self.log.info("task::task %r arrived on %r", msg_id, eid)
         if msg_id in self.unassigned:
             self.unassigned.remove(msg_id)
         # else:
         #     self.log.debug("task::task %r not listed as MIA?!"%(msg_id))
 
         self.tasks[eid].append(msg_id)
         # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid))
         try:
             self.db.update_record(msg_id, dict(engine_uuid=engine_uuid))
         except Exception:
             self.log.error("DB Error saving task destination %r", msg_id, exc_info=True)
 
 
     def mia_task_request(self, idents, msg):
         raise NotImplementedError
         client_id = idents[0]
         # content = dict(mia=self.mia,status='ok')
         # self.session.send('mia_reply', content=content, idents=client_id)
 
 
     #--------------------- IOPub Traffic ------------------------------
 
     def save_iopub_message(self, topics, msg):
         """save an iopub message into the db"""
         # print (topics)
         try:
             msg = self.session.unserialize(msg, content=True)
         except Exception:
             self.log.error("iopub::invalid IOPub message", exc_info=True)
             return
 
         parent = msg['parent_header']
         if not parent:
             self.log.warn("iopub::IOPub message lacks parent: %r", msg)
             return
         msg_id = parent['msg_id']
         msg_type = msg['header']['msg_type']
         content = msg['content']
 
         # ensure msg_id is in db
         try:
             rec = self.db.get_record(msg_id)
         except KeyError:
             rec = empty_record()
             rec['msg_id'] = msg_id
             self.db.add_record(msg_id, rec)
         # stream
         d = {}
         if msg_type == 'stream':
             name = content['name']
             s = rec[name] or ''
             d[name] = s + content['data']
 
         elif msg_type == 'pyerr':
             d['pyerr'] = content
         elif msg_type == 'pyin':
             d['pyin'] = content['code']
         elif msg_type in ('display_data', 'pyout'):
             d[msg_type] = content
         elif msg_type == 'status':
             pass
         elif msg_type == 'data_pub':
             self.log.info("ignored data_pub message for %s" % msg_id)
         else:
             self.log.warn("unhandled iopub msg_type: %r", msg_type)
 
         if not d:
             return
 
         try:
             self.db.update_record(msg_id, d)
         except Exception:
             self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True)
 
 
 
     #-------------------------------------------------------------------------
     # Registration requests
     #-------------------------------------------------------------------------
 
     def connection_request(self, client_id, msg):
         """Reply with connection addresses for clients."""
         self.log.info("client::client %r connected", client_id)
         content = dict(status='ok')
         jsonable = {}
         for k,v in self.keytable.iteritems():
             if v not in self.dead_engines:
                 jsonable[str(k)] = v
         content['engines'] = jsonable
         self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)
 
     def register_engine(self, reg, msg):
         """Register a new engine."""
         content = msg['content']
         try:
             uuid = content['uuid']
         except KeyError:
             self.log.error("registration::queue not specified", exc_info=True)
             return
 
         eid = self._next_id
 
         self.log.debug("registration::register_engine(%i, %r)", eid, uuid)
 
         content = dict(id=eid,status='ok',hb_period=self.heartmonitor.period)
         # check if requesting available IDs:
         if cast_bytes(uuid) in self.by_ident:
             try:
                 raise KeyError("uuid %r in use" % uuid)
             except:
                 content = error.wrap_exception()
                 self.log.error("uuid %r in use", uuid, exc_info=True)
         else:
             for h, ec in self.incoming_registrations.iteritems():
                 if uuid == h:
                     try:
                         raise KeyError("heart_id %r in use" % uuid)
                     except:
                         self.log.error("heart_id %r in use", uuid, exc_info=True)
                         content = error.wrap_exception()
                     break
                 elif uuid == ec.uuid:
                     try:
                         raise KeyError("uuid %r in use" % uuid)
                     except:
                         self.log.error("uuid %r in use", uuid, exc_info=True)
                         content = error.wrap_exception()
                     break
 
         msg = self.session.send(self.query, "registration_reply",
                 content=content,
                 ident=reg)
 
         heart = cast_bytes(uuid)
 
         if content['status'] == 'ok':
             if heart in self.heartmonitor.hearts:
                 # already beating
                 self.incoming_registrations[heart] = EngineConnector(id=eid,uuid=uuid)
                 self.finish_registration(heart)
             else:
                 purge = lambda : self._purge_stalled_registration(heart)
                 dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                 dc.start()
                 self.incoming_registrations[heart] = EngineConnector(id=eid,uuid=uuid,stallback=dc)
         else:
             self.log.error("registration::registration %i failed: %r", eid, content['evalue'])
         
         return eid
 
     def unregister_engine(self, ident, msg):
         """Unregister an engine that explicitly requested to leave."""
         try:
             eid = msg['content']['id']
         except:
             self.log.error("registration::bad engine id for unregistration: %r", ident, exc_info=True)
             return
         self.log.info("registration::unregister_engine(%r)", eid)
         # print (eid)
         uuid = self.keytable[eid]
         content=dict(id=eid, uuid=uuid)
         self.dead_engines.add(uuid)
         # self.ids.remove(eid)
         # uuid = self.keytable.pop(eid)
         #
         # ec = self.engines.pop(eid)
         # self.hearts.pop(ec.heartbeat)
         # self.by_ident.pop(ec.queue)
         # self.completed.pop(eid)
         handleit = lambda : self._handle_stranded_msgs(eid, uuid)
         dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop)
         dc.start()
         ############## TODO: HANDLE IT ################
         
         self._save_engine_state()
 
         if self.notifier:
             self.session.send(self.notifier, "unregistration_notification", content=content)
 
     def _handle_stranded_msgs(self, eid, uuid):
         """Handle messages known to be on an engine when the engine unregisters.
 
         It is possible that this will fire prematurely - that is, an engine will
         go down after completing a result, and the client will be notified
         that the result failed and later receive the actual result.
         """
 
         outstanding = self.queues[eid]
 
         for msg_id in outstanding:
             self.pending.remove(msg_id)
             self.all_completed.add(msg_id)
             try:
                 raise error.EngineError("Engine %r died while running task %r" % (eid, msg_id))
             except:
                 content = error.wrap_exception()
             # build a fake header:
             header = {}
             header['engine'] = uuid
             header['date'] = datetime.now()
             rec = dict(result_content=content, result_header=header, result_buffers=[])
             rec['completed'] = header['date']
             rec['engine_uuid'] = uuid
             try:
                 self.db.update_record(msg_id, rec)
             except Exception:
                 self.log.error("DB Error handling stranded msg %r", msg_id, exc_info=True)
 
 
     def finish_registration(self, heart):
         """Second half of engine registration, called after our HeartMonitor
         has received a beat from the Engine's Heart."""
         try:
             ec = self.incoming_registrations.pop(heart)
         except KeyError:
             self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
             return
         self.log.info("registration::finished registering engine %i:%s", ec.id, ec.uuid)
         if ec.stallback is not None:
             ec.stallback.stop()
         eid = ec.id
         self.ids.add(eid)
         self.keytable[eid] = ec.uuid
         self.engines[eid] = ec
         self.by_ident[cast_bytes(ec.uuid)] = ec.id
         self.queues[eid] = list()
         self.tasks[eid] = list()
         self.completed[eid] = list()
         self.hearts[heart] = eid
         content = dict(id=eid, uuid=self.engines[eid].uuid)
         if self.notifier:
             self.session.send(self.notifier, "registration_notification", content=content)
         self.log.info("engine::Engine Connected: %i", eid)
         
         self._save_engine_state()
 
     def _purge_stalled_registration(self, heart):
         if heart in self.incoming_registrations:
             ec = self.incoming_registrations.pop(heart)
             self.log.info("registration::purging stalled registration: %i", ec.id)
         else:
             pass
 
     #-------------------------------------------------------------------------
     # Engine State
     #-------------------------------------------------------------------------
 
 
     def _cleanup_engine_state_file(self):
         """cleanup engine state mapping"""
         
         if os.path.exists(self.engine_state_file):
             self.log.debug("cleaning up engine state: %s", self.engine_state_file)
             try:
                 os.remove(self.engine_state_file)
             except IOError:
                 self.log.error("Couldn't cleanup file: %s", self.engine_state_file, exc_info=True)
 
 
     def _save_engine_state(self):
         """save engine mapping to JSON file"""
         if not self.engine_state_file:
             return
         self.log.debug("save engine state to %s" % self.engine_state_file)
         state = {}
         engines = {}
         for eid, ec in self.engines.iteritems():
             if ec.uuid not in self.dead_engines:
                 engines[eid] = ec.uuid
         
         state['engines'] = engines
         
         state['next_id'] = self._idcounter
         
         with open(self.engine_state_file, 'w') as f:
             json.dump(state, f)
 
 
     def _load_engine_state(self):
         """load engine mapping from JSON file"""
         if not os.path.exists(self.engine_state_file):
             return
         
         self.log.info("loading engine state from %s" % self.engine_state_file)
         
         with open(self.engine_state_file) as f:
             state = json.load(f)
         
         save_notifier = self.notifier
         self.notifier = None
         for eid, uuid in state['engines'].iteritems():
             heart = uuid.encode('ascii')
             # start with this heart as current and beating:
             self.heartmonitor.responses.add(heart)
             self.heartmonitor.hearts.add(heart)
             
             self.incoming_registrations[heart] = EngineConnector(id=int(eid), uuid=uuid)
             self.finish_registration(heart)
         
         self.notifier = save_notifier
         
         self._idcounter = state['next_id']
 
     #-------------------------------------------------------------------------
     # Client Requests
     #-------------------------------------------------------------------------
 
     def shutdown_request(self, client_id, msg):
         """handle shutdown request."""
         self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
         # also notify other clients of shutdown
         self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
         dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
         dc.start()
 
     def _shutdown(self):
         self.log.info("hub::hub shutting down.")
         time.sleep(0.1)
         sys.exit(0)
 
 
     def check_load(self, client_id, msg):
         content = msg['content']
         try:
             targets = content['targets']
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error",
                     content=content, ident=client_id)
             return
 
         content = dict(status='ok')
         # loads = {}
         for t in targets:
             content[bytes(t)] = len(self.queues[t])+len(self.tasks[t])
         self.session.send(self.query, "load_reply", content=content, ident=client_id)
 
 
     def queue_status(self, client_id, msg):
         """Return the Queue status of one or more targets.
         if verbose: return the msg_ids
         else: return len of each type.
         keys: queue (pending MUX jobs)
             tasks (pending Task jobs)
             completed (finished jobs from both queues)"""
         content = msg['content']
         targets = content['targets']
         try:
             targets = self._validate_targets(targets)
         except:
             content = error.wrap_exception()
             self.session.send(self.query, "hub_error",
                     content=content, ident=client_id)
             return
         verbose = content.get('verbose', False)
         content = dict(status='ok')
         for t in targets:
             queue = self.queues[t]
             completed = self.completed[t]
             tasks = self.tasks[t]
             if not verbose:
                 queue = len(queue)
                 completed = len(completed)
                 tasks = len(tasks)
             content[str(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
         content['unassigned'] = list(self.unassigned) if verbose else len(self.unassigned)
         # print (content)
         self.session.send(self.query, "queue_reply", content=content, ident=client_id)
 
     def purge_results(self, client_id, msg):
         """Purge results from memory. This method is more valuable before we move
         to a DB based message storage mechanism."""
         content = msg['content']
         self.log.info("Dropping records with %s", content)
         msg_ids = content.get('msg_ids', [])
         reply = dict(status='ok')
         if msg_ids == 'all':
             try:
                 self.db.drop_matching_records(dict(completed={'$ne':None}))
             except Exception:
                 reply = error.wrap_exception()
         else:
             pending = filter(lambda m: m in self.pending, msg_ids)
             if pending:
                 try:
                     raise IndexError("msg pending: %r" % pending[0])
                 except:
                     reply = error.wrap_exception()
             else:
                 try:
                     self.db.drop_matching_records(dict(msg_id={'$in':msg_ids}))
                 except Exception:
                     reply = error.wrap_exception()
 
             if reply['status'] == 'ok':
                 eids = content.get('engine_ids', [])
                 for eid in eids:
                     if eid not in self.engines:
                         try:
                             raise IndexError("No such engine: %i" % eid)
                         except:
                             reply = error.wrap_exception()
                         break
                     uid = self.engines[eid].uuid
                     try:
                         self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
                     except Exception:
                         reply = error.wrap_exception()
                         break
 
         self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)
 
     def resubmit_task(self, client_id, msg):
         """Resubmit one or more tasks."""
         def finish(reply):
             self.session.send(self.query, 'resubmit_reply', content=reply, ident=client_id)
 
         content = msg['content']
         msg_ids = content['msg_ids']
         reply = dict(status='ok')
         try:
             records = self.db.find_records({'msg_id' : {'$in' : msg_ids}}, keys=[
                 'header', 'content', 'buffers'])
         except Exception:
             self.log.error('db::db error finding tasks to resubmit', exc_info=True)
             return finish(error.wrap_exception())
 
         # validate msg_ids
         found_ids = [ rec['msg_id'] for rec in records ]
         pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ]
         if len(records) > len(msg_ids):
             try:
                 raise RuntimeError("DB appears to be in an inconsistent state."
                     "More matching records were found than should exist")
             except Exception:
                 return finish(error.wrap_exception())
         elif len(records) < len(msg_ids):
             missing = [ m for m in msg_ids if m not in found_ids ]
             try:
                 raise KeyError("No such msg(s): %r" % missing)
             except KeyError:
                 return finish(error.wrap_exception())
         elif pending_ids:
             pass
             # no need to raise on resubmit of pending task, now that we
             # resubmit under new ID, but do we want to raise anyway?
             # msg_id = invalid_ids[0]
             # try:
             #     raise ValueError("Task(s) %r appears to be inflight" % )
             # except Exception:
             #     return finish(error.wrap_exception())
 
         # mapping of original IDs to resubmitted IDs
         resubmitted = {}
 
         # send the messages
         for rec in records:
             header = rec['header']
             msg = self.session.msg(header['msg_type'], parent=header)
             msg_id = msg['msg_id']
             msg['content'] = rec['content']
             
             # use the old header, but update msg_id and timestamp
             fresh = msg['header']
             header['msg_id'] = fresh['msg_id']
             header['date'] = fresh['date']
             msg['header'] = header
 
             self.session.send(self.resubmit, msg, buffers=rec['buffers'])
 
             resubmitted[rec['msg_id']] = msg_id
             self.pending.add(msg_id)
             msg['buffers'] = rec['buffers']
             try:
                 self.db.add_record(msg_id, init_record(msg))
             except Exception:
                 self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True)
                 return finish(error.wrap_exception())
 
         finish(dict(status='ok', resubmitted=resubmitted))
         
         # store the new IDs in the Task DB
         for msg_id, resubmit_id in resubmitted.iteritems():
             try:
                 self.db.update_record(msg_id, {'resubmitted' : resubmit_id})
             except Exception:
                 self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True)
 
 
     def _extract_record(self, rec):
         """decompose a TaskRecord dict into subsection of reply for get_result"""
         io_dict = {}
         for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'):
                 io_dict[key] = rec[key]
         content = { 
             'header': rec['header'],
             'metadata': rec['metadata'],
             'result_metadata': rec['result_metadata'],
             'result_header' : rec['result_header'],
             'result_content': rec['result_content'],
             'received' : rec['received'],
             'io' : io_dict,
         }
         if rec['result_buffers']:
             buffers = map(bytes, rec['result_buffers'])
         else:
             buffers = []
 
         return content, buffers
 
     def get_results(self, client_id, msg):
         """Get the result of 1 or more messages."""
         content = msg['content']
         msg_ids = sorted(set(content['msg_ids']))
         statusonly = content.get('status_only', False)
         pending = []
         completed = []
         content = dict(status='ok')
         content['pending'] = pending
         content['completed'] = completed
         buffers = []
         if not statusonly:
             try:
                 matches = self.db.find_records(dict(msg_id={'$in':msg_ids}))
                 # turn match list into dict, for faster lookup
                 records = {}
                 for rec in matches:
                     records[rec['msg_id']] = rec
             except Exception:
                 content = error.wrap_exception()
                 self.session.send(self.query, "result_reply", content=content,
                                                     parent=msg, ident=client_id)
                 return
         else:
             records = {}
         for msg_id in msg_ids:
             if msg_id in self.pending:
                 pending.append(msg_id)
             elif msg_id in self.all_completed:
                 completed.append(msg_id)
                 if not statusonly:
                     c,bufs = self._extract_record(records[msg_id])
                     content[msg_id] = c
                     buffers.extend(bufs)
             elif msg_id in records:
                 if rec['completed']:
                     completed.append(msg_id)
                     c,bufs = self._extract_record(records[msg_id])
                     content[msg_id] = c
                     buffers.extend(bufs)
                 else:
                     pending.append(msg_id)
             else:
                 try:
                     raise KeyError('No such message: '+msg_id)
                 except:
                     content = error.wrap_exception()
                 break
         self.session.send(self.query, "result_reply", content=content,
                                             parent=msg, ident=client_id,
                                             buffers=buffers)
 
     def get_history(self, client_id, msg):
         """Get a list of all msg_ids in our DB records"""
         try:
             msg_ids = self.db.get_history()
         except Exception as e:
             content = error.wrap_exception()
         else:
             content = dict(status='ok', history=msg_ids)
 
         self.session.send(self.query, "history_reply", content=content,
                                             parent=msg, ident=client_id)
 
     def db_query(self, client_id, msg):
         """Perform a raw query on the task record database."""
         content = msg['content']
         query = content.get('query', {})
         keys = content.get('keys', None)
         buffers = []
         empty = list()
         try:
             records = self.db.find_records(query, keys)
         except Exception as e:
             content = error.wrap_exception()
         else:
             # extract buffers from reply content:
             if keys is not None:
                 buffer_lens = [] if 'buffers' in keys else None
                 result_buffer_lens = [] if 'result_buffers' in keys else None
             else:
                 buffer_lens = None
                 result_buffer_lens = None
 
             for rec in records:
                 # buffers may be None, so double check
                 b = rec.pop('buffers', empty) or empty
                 if buffer_lens is not None:
                     buffer_lens.append(len(b))
                     buffers.extend(b)
                 rb = rec.pop('result_buffers', empty) or empty
                 if result_buffer_lens is not None:
                     result_buffer_lens.append(len(rb))
                     buffers.extend(rb)
             content = dict(status='ok', records=records, buffer_lens=buffer_lens,
                                     result_buffer_lens=result_buffer_lens)
         # self.log.debug (content)
         self.session.send(self.query, "db_reply", content=content,
                                             parent=msg, ident=client_id,
                                             buffers=buffers)
 

IPython/qt/console/qtconsoleapp.py

@@ -1,382 +1,374 b''
 """ A minimal application using the Qt console-style IPython frontend.
 
 This is not a complete console app, as subprocess will not be able to receive
 input, there is no real readline support, among other limitations.
 
 Authors:
 
 * Evan Patterson
 * Min RK
 * Erik Tollerud
 * Fernando Perez
 * Bussonnier Matthias
 * Thomas Kluyver
 * Paul Ivanov
 
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # stdlib imports
 import os
 import signal
 import sys
 
 # If run on Windows, install an exception hook which pops up a
 # message box. Pythonw.exe hides the console, so without this
 # the application silently fails to load.
 #
 # We always install this handler, because the expectation is for
 # qtconsole to bring up a GUI even if called from the console.
 # The old handler is called, so the exception is printed as well.
 # If desired, check for pythonw with an additional condition
 # (sys.executable.lower().find('pythonw.exe') >= 0).
 if os.name == 'nt':
     old_excepthook = sys.excepthook
 
     def gui_excepthook(exctype, value, tb):
         try:
             import ctypes, traceback
             MB_ICONERROR = 0x00000010L
             title = u'Error starting IPython QtConsole'
             msg = u''.join(traceback.format_exception(exctype, value, tb))
             ctypes.windll.user32.MessageBoxW(0, msg, title, MB_ICONERROR)
         finally:
             # Also call the old exception hook to let it do
             # its thing too.
             old_excepthook(exctype, value, tb)
 
     sys.excepthook = gui_excepthook
 
 # System library imports
 from IPython.external.qt import QtCore, QtGui
 
 # Local imports
 from IPython.config.application import catch_config_error
 from IPython.core.application import BaseIPythonApplication
 from IPython.qt.console.ipython_widget import IPythonWidget
 from IPython.qt.console.rich_ipython_widget import RichIPythonWidget
 from IPython.qt.console import styles
 from IPython.qt.console.mainwindow import MainWindow
 from IPython.qt.client import QtKernelClient
 from IPython.qt.manager import QtKernelManager
 from IPython.utils.traitlets import (
     Dict, Unicode, CBool, Any
 )
 
 from IPython.consoleapp import (
         IPythonConsoleApp, app_aliases, app_flags, flags, aliases
     )
 
 #-----------------------------------------------------------------------------
 # Network Constants
 #-----------------------------------------------------------------------------
 
 from IPython.utils.localinterfaces import is_local_ip
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython qtconsole                      # start the qtconsole
 ipython qtconsole --matplotlib=inline  # start with matplotlib inline plotting mode
 """
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 # start with copy of flags
 flags = dict(flags)
 qt_flags = {
     'plain' : ({'IPythonQtConsoleApp' : {'plain' : True}},
             "Disable rich text support."),
 }
 
 # and app_flags from the Console Mixin
 qt_flags.update(app_flags)
 # add frontend flags to the full set
 flags.update(qt_flags)
 
 # start with copy of front&backend aliases list
 aliases = dict(aliases)
 qt_aliases = dict(
     style = 'IPythonWidget.syntax_style',
     stylesheet = 'IPythonQtConsoleApp.stylesheet',
     colors = 'ZMQInteractiveShell.colors',
 
     editor = 'IPythonWidget.editor',
     paging = 'ConsoleWidget.paging',
 )
 # and app_aliases from the Console Mixin
 qt_aliases.update(app_aliases)
 qt_aliases.update({'gui-completion':'ConsoleWidget.gui_completion'})
 # add frontend aliases to the full set
 aliases.update(qt_aliases)
 
 # get flags&aliases into sets, and remove a couple that
 # shouldn't be scrubbed from backend flags:
 qt_aliases = set(qt_aliases.keys())
 qt_aliases.remove('colors')
 qt_flags = set(qt_flags.keys())
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # IPythonQtConsole
 #-----------------------------------------------------------------------------
 
 
 class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp):
     name = 'ipython-qtconsole'
 
     description = """
         The IPython QtConsole.
         
         This launches a Console-style application using Qt.  It is not a full
         console, in that launched terminal subprocesses will not be able to accept
         input.
         
         The QtConsole supports various extra features beyond the Terminal IPython
         shell, such as inline plotting with matplotlib, via:
         
             ipython qtconsole --matplotlib=inline
         
         as well as saving your session as HTML, and printing the output.
         
     """
     examples = _examples
 
     classes = [IPythonWidget] + IPythonConsoleApp.classes
     flags = Dict(flags)
     aliases = Dict(aliases)
     frontend_flags = Any(qt_flags)
     frontend_aliases = Any(qt_aliases)
     kernel_client_class = QtKernelClient
     kernel_manager_class = QtKernelManager
 
     stylesheet = Unicode('', config=True,
         help="path to a custom CSS stylesheet")
 
     hide_menubar = CBool(False, config=True,
         help="Start the console window with the menu bar hidden.")
 
     maximize = CBool(False, config=True,
         help="Start the console window maximized.")
 
     plain = CBool(False, config=True,
         help="Use a plaintext widget instead of rich text (plain can't print/save).")
 
     def _plain_changed(self, name, old, new):
         kind = 'plain' if new else 'rich'
         self.config.ConsoleWidget.kind = kind
         if new:
             self.widget_factory = IPythonWidget
         else:
             self.widget_factory = RichIPythonWidget
 
     # the factory for creating a widget
     widget_factory = Any(RichIPythonWidget)
 
     def parse_command_line(self, argv=None):
         super(IPythonQtConsoleApp, self).parse_command_line(argv)
         self.build_kernel_argv(argv)
 
 
     def new_frontend_master(self):
         """ Create and return new frontend attached to new kernel, launched on localhost.
         """
         kernel_manager = self.kernel_manager_class(
                                 connection_file=self._new_connection_file(),
                                 parent=self,
                                 autorestart=True,
         )
         # start the kernel
         kwargs = dict()
         kwargs['extra_arguments'] = self.kernel_argv
         kernel_manager.start_kernel(**kwargs)
         kernel_manager.client_factory = self.kernel_client_class
         kernel_client = kernel_manager.client()
         kernel_client.start_channels(shell=True, iopub=True)
         widget = self.widget_factory(config=self.config,
                                    local_kernel=True)
         self.init_colors(widget)
         widget.kernel_manager = kernel_manager
         widget.kernel_client = kernel_client
         widget._existing = False
         widget._may_close = True
         widget._confirm_exit = self.confirm_exit
         return widget
 
     def new_frontend_slave(self, current_widget):
         """Create and return a new frontend attached to an existing kernel.
         
         Parameters
         ----------
         current_widget : IPythonWidget
             The IPythonWidget whose kernel this frontend is to share
         """
         kernel_client = self.kernel_client_class(
                                 connection_file=current_widget.kernel_client.connection_file,
                                 config = self.config,
         )
         kernel_client.load_connection_file()
         kernel_client.start_channels()
         widget = self.widget_factory(config=self.config,
                                 local_kernel=False)
         self.init_colors(widget)
         widget._existing = True
         widget._may_close = False
         widget._confirm_exit = False
         widget.kernel_client = kernel_client
         widget.kernel_manager = current_widget.kernel_manager
         return widget
 
     def init_qt_app(self):
         # separate from qt_elements, because it must run first
         self.app = QtGui.QApplication([])
 
     def init_qt_elements(self):
         # Create the widget.
 
         base_path = os.path.abspath(os.path.dirname(__file__))
         icon_path = os.path.join(base_path, 'resources', 'icon', 'IPythonConsole.svg')
         self.app.icon = QtGui.QIcon(icon_path)
         QtGui.QApplication.setWindowIcon(self.app.icon)
 
         ip = self.ip
         local_kernel = (not self.existing) or is_local_ip(ip)
         self.widget = self.widget_factory(config=self.config,
                                         local_kernel=local_kernel)
         self.init_colors(self.widget)
         self.widget._existing = self.existing
         self.widget._may_close = not self.existing
         self.widget._confirm_exit = self.confirm_exit
 
         self.widget.kernel_manager = self.kernel_manager
         self.widget.kernel_client = self.kernel_client
         self.window = MainWindow(self.app,
                                 confirm_exit=self.confirm_exit,
                                 new_frontend_factory=self.new_frontend_master,
                                 slave_frontend_factory=self.new_frontend_slave,
                                 )
         self.window.log = self.log
         self.window.add_tab_with_frontend(self.widget)
         self.window.init_menu_bar()
 
         # Ignore on OSX, where there is always a menu bar
         if sys.platform != 'darwin' and self.hide_menubar:
             self.window.menuBar().setVisible(False)
 
         self.window.setWindowTitle('IPython')
 
     def init_colors(self, widget):
         """Configure the coloring of the widget"""
         # Note: This will be dramatically simplified when colors
         # are removed from the backend.
 
         # parse the colors arg down to current known labels
-        try:
-            colors = self.config.ZMQInteractiveShell.colors
-        except AttributeError:
-            colors = None
-        try:
-            style = self.config.IPythonWidget.syntax_style
-        except AttributeError:
-            style = None
-        try:
-            sheet = self.config.IPythonWidget.style_sheet
-        except AttributeError:
-            sheet = None
+        cfg = self.config
+        colors = cfg.ZMQInteractiveShell.colors if 'ZMQInteractiveShell.colors' in cfg else None
+        style = cfg.IPythonWidget.syntax_style if 'IPythonWidget.syntax_style' in cfg else None
+        sheet = cfg.IPythonWidget.style_sheet if 'IPythonWidget.style_sheet' in cfg else None
 
         # find the value for colors:
         if colors:
             colors=colors.lower()
             if colors in ('lightbg', 'light'):
                 colors='lightbg'
             elif colors in ('dark', 'linux'):
                 colors='linux'
             else:
                 colors='nocolor'
         elif style:
             if style=='bw':
                 colors='nocolor'
             elif styles.dark_style(style):
                 colors='linux'
             else:
                 colors='lightbg'
         else:
             colors=None
 
         # Configure the style
         if style:
             widget.style_sheet = styles.sheet_from_template(style, colors)
             widget.syntax_style = style
             widget._syntax_style_changed()
             widget._style_sheet_changed()
         elif colors:
             # use a default dark/light/bw style
             widget.set_default_style(colors=colors)
 
         if self.stylesheet:
             # we got an explicit stylesheet
             if os.path.isfile(self.stylesheet):
                 with open(self.stylesheet) as f:
                     sheet = f.read()
             else:
                 raise IOError("Stylesheet %r not found." % self.stylesheet)
         if sheet:
             widget.style_sheet = sheet
             widget._style_sheet_changed()
             
 
     def init_signal(self):
         """allow clean shutdown on sigint"""
         signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2))
         # need a timer, so that QApplication doesn't block until a real
         # Qt event fires (can require mouse movement)
         # timer trick from http://stackoverflow.com/q/4938723/938949
         timer = QtCore.QTimer()
          # Let the interpreter run each 200 ms:
         timer.timeout.connect(lambda: None)
         timer.start(200)
         # hold onto ref, so the timer doesn't get cleaned up
         self._sigint_timer = timer
 
     @catch_config_error
     def initialize(self, argv=None):
         self.init_qt_app()
         super(IPythonQtConsoleApp, self).initialize(argv)
         IPythonConsoleApp.initialize(self,argv)
         self.init_qt_elements()
         self.init_signal()
 
     def start(self):
 
         # draw the window
         if self.maximize:
             self.window.showMaximized()
         else:
             self.window.show()
         self.window.raise_()
 
         # Start the application main loop.
         self.app.exec_()
 
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 def main():
     app = IPythonQtConsoleApp()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()